com.amazonaws.services.s3.S3ClientOptions Java Examples

The following examples show how to use com.amazonaws.services.s3.S3ClientOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: S3StorageDriver.java From dcos-cassandra-service with Apache License 2.0

6 votes

private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException {
    final String accessKey = ctx.getAccountId();
    final String secretKey = ctx.getSecretKey();
    String endpoint = getEndpoint(ctx);
    LOGGER.info("endpoint: {}", endpoint);

    final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey);
    final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials);
    amazonS3Client.setEndpoint(endpoint);

    if (ctx.usesEmc()) {
        final S3ClientOptions options = new S3ClientOptions();
        options.setPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(options);
    }

    return amazonS3Client;
}

Example #2

Source File: S3StorageFactory.java From digdag with Apache License 2.0

6 votes

@Override
public Storage newStorage(Config config)
{
    AmazonS3Client client = new AmazonS3Client(
            buildCredentialsProvider(config),
            buildClientConfiguration(config));
    if (config.has("endpoint")) {
        client.setEndpoint(config.get("endpoint", String.class));
    }

    if (config.has("path-style-access")) {
        client.setS3ClientOptions(
          S3ClientOptions.builder().setPathStyleAccess(
            config.get("path-style-access", Boolean.class, false)
          ).build());
    }

    String bucket = config.get("bucket", String.class);

    return new S3Storage(config, client, bucket);
}

Example #3

Source File: AbstractS3Processor.java From localization_nifi with Apache License 2.0

5 votes

private void initalizeEndpointOverride(final ProcessContext context, final AmazonS3Client s3) {
    // if ENDPOINT_OVERRIDE is set, use PathStyleAccess
    if(StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).getValue()).isEmpty() == false){
        final S3ClientOptions s3Options = new S3ClientOptions();
        s3Options.setPathStyleAccess(true);
        s3.setS3ClientOptions(s3Options);
    }
}

Example #4

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

5 votes

Boolean LifeCycleStatus(String access_key, String secret_key, String bucket, String endpoint, Boolean enable) {
    String message = null;
    boolean result = false;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    try {
        message = s3Client.getBucketLifecycleConfiguration(bucket).getRules().toString();
        if (message == null) {
            result = false;
        } else {
            result = true;
        }
    } catch (Exception lifecyclestatus) {
    }

    return result;
}

Example #5

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

5 votes

Boolean VersioningStatus(String access_key, String secret_key, String bucket, String endpoint, Boolean enable) {
    String message = null;
    boolean result = false;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());

    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    try {
        message = s3Client.getBucketVersioningConfiguration(bucket).getStatus().toString();
        if (message.contains("Enabled") || message.contains("Suspended")) {
            result = true;
        } else {
            result = false;
        }
    } catch (Exception versioning) {
    }

    return result;
}

Example #6

Source File: AbstractS3Processor.java From nifi with Apache License 2.0

5 votes

private void initalizeEndpointOverride(final ProcessContext context, final AmazonS3Client s3) {
    // if ENDPOINT_OVERRIDE is set, use PathStyleAccess
    if(StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).evaluateAttributeExpressions().getValue()).isEmpty() == false){
        final S3ClientOptions s3Options = new S3ClientOptions();
        s3Options.setPathStyleAccess(true);
        s3.setS3ClientOptions(s3Options);
    }
}

Example #7

Source File: S3ClientFactory.java From front50 with Apache License 2.0

5 votes

public static AmazonS3 create(
    AWSCredentialsProvider awsCredentialsProvider, S3Properties s3Properties) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  if (s3Properties.getProxyProtocol() != null) {
    if (s3Properties.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
      clientConfiguration.setProtocol(Protocol.HTTPS);
    } else {
      clientConfiguration.setProtocol(Protocol.HTTP);
    }
    Optional.ofNullable(s3Properties.getProxyHost()).ifPresent(clientConfiguration::setProxyHost);
    Optional.ofNullable(s3Properties.getProxyPort())
        .map(Integer::parseInt)
        .ifPresent(clientConfiguration::setProxyPort);
  }

  AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

  if (!StringUtils.isEmpty(s3Properties.getEndpoint())) {
    client.setEndpoint(s3Properties.getEndpoint());

    if (!StringUtils.isEmpty(s3Properties.getRegionOverride())) {
      client.setSignerRegionOverride(s3Properties.getRegionOverride());
    }

    client.setS3ClientOptions(
        S3ClientOptions.builder().setPathStyleAccess(s3Properties.getPathStyleAccess()).build());
  } else {
    Optional.ofNullable(s3Properties.getRegion())
        .map(Regions::fromName)
        .map(Region::getRegion)
        .ifPresent(client::setRegion);
  }

  return client;
}

Example #8

Source File: S3PersistWriter.java From streams with Apache License 2.0

5 votes

@Override
public void prepare(Object configurationObject) {

  lineWriterUtil = LineReadWriteUtil.getInstance(s3WriterConfiguration);

  // Connect to S3
  synchronized (this) {

    try {
      // if the user has chosen to not set the object mapper, then set a default object mapper for them.
      if (this.objectMapper == null) {
        this.objectMapper = StreamsJacksonMapper.getInstance();
      }

      // Create the credentials Object
      if (this.amazonS3Client == null) {
        AWSCredentials credentials = new BasicAWSCredentials(s3WriterConfiguration.getKey(), s3WriterConfiguration.getSecretKey());

        ClientConfiguration clientConfig = new ClientConfiguration();
        clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toString()));

        // We do not want path style access
        S3ClientOptions clientOptions = new S3ClientOptions();
        clientOptions.setPathStyleAccess(false);

        this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
        if (StringUtils.isNotEmpty(s3WriterConfiguration.getRegion())) {
          this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3WriterConfiguration.getRegion())));
        }
        this.amazonS3Client.setS3ClientOptions(clientOptions);
      }
    } catch (Exception ex) {
      LOGGER.error("Exception while preparing the S3 client: {}", ex);
    }

    Preconditions.checkArgument(this.amazonS3Client != null);
  }
}

Example #9

Source File: Acl.java From cloudExplorer with GNU General Public License v3.0

5 votes

void setBUCKETwebsite(String object, String access_key, String secret_key, String endpoint, String bucket) {
    try {
        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration());
        if (endpoint.contains("amazonaws.com")) {
            String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
            if (aws_endpoint.contains("US")) {
                s3Client.setEndpoint("https://s3.amazonaws.com");
            } else if (aws_endpoint.contains("us-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("eu-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("ap-")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("sa-east-1")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else {
                s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
            }
        } else {
            s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
            s3Client.setEndpoint(endpoint);
        }
        BucketWebsiteConfiguration bucketWebsiteConfiguration = s3Client.getBucketWebsiteConfiguration(bucket);
        s3Client.setBucketAcl(bucket, CannedAccessControlList.PublicRead);
        s3Client.setBucketWebsiteConfiguration(bucket, new BucketWebsiteConfiguration("index.html", "error.html"));
    } catch (Exception setACLpublic) {
        mainFrame.jTextArea1.append("\nException occurred in ACL");
    }
}

Example #10

Source File: Acl.java From cloudExplorer with GNU General Public License v3.0

5 votes

String setACLurl(String object, String access_key, String secret_key, String endpoint, String bucket) {
    String URL = null;
    try {
        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration());
         if (endpoint.contains("amazonaws.com")) {
            String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
            if (aws_endpoint.contains("US")) {
                s3Client.setEndpoint("https://s3.amazonaws.com");
            } else if (aws_endpoint.contains("us-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("eu-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("ap-")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("sa-east-1")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else {
                s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
            }
        } else {
            s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
            s3Client.setEndpoint(endpoint);
        }
        java.util.Date expiration = new java.util.Date();
        long milliSeconds = expiration.getTime();
        milliSeconds += 1000 * 60 * 1000; // Add 1 hour.
        expiration.setTime(milliSeconds);
        GeneratePresignedUrlRequest generatePresignedUrlRequest
                = new GeneratePresignedUrlRequest(bucket, object);
        generatePresignedUrlRequest.setMethod(HttpMethod.GET);
        generatePresignedUrlRequest.setExpiration(expiration);
        URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
        URL = ("Pre-Signed URL = " + url.toString());
        StringSelection stringSelection = new StringSelection(url.toString());
    } catch (Exception setACLpublic) {
    }
    return URL;
}

Example #11

Source File: S3WaitOperatorFactoryTest.java From digdag with Apache License 2.0

5 votes

@Test
public void testCustomRegionAndPathStyleAccess()
        throws Exception
{
    Config config = newConfig();
    config.set("path_style_access", true);
    config.set("_command", BUCKET + "/" + KEY);
    when(taskRequest.getConfig()).thenReturn(config);

    when(s3Secrets.getSecretOptional("region")).thenReturn(Optional.of(REGION));

    when(s3Client.getObjectMetadata(objectMetadataRequestCaptor.capture())).thenThrow(NOT_FOUND_EXCEPTION);

    Operator operator = factory.newOperator(newContext(projectPath, taskRequest));

    try {
        operator.run();
        fail();
    }
    catch (TaskExecutionException ignore) {
    }

    verify(s3Client).setS3ClientOptions(s3ClientOptionsCaptor.capture());
    S3ClientOptions s3ClientOptions = s3ClientOptionsCaptor.getValue();
    assertThat(s3ClientOptions.isPathStyleAccess(), is(true));

    verify(s3Client).setRegion(RegionUtils.getRegion(REGION));
}

Example #12

Source File: TestPrestoS3FileSystem.java From presto with Apache License 2.0

5 votes

@Test
public void testPathStyleAccess()
        throws Exception
{
    Configuration config = new Configuration(false);
    config.setBoolean(S3_PATH_STYLE_ACCESS, true);

    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        fs.initialize(new URI("s3n://test-bucket/"), config);
        S3ClientOptions clientOptions = getFieldValue(fs.getS3Client(), AmazonS3Client.class, "clientOptions", S3ClientOptions.class);
        assertTrue(clientOptions.isPathStyleAccess());
    }
}

Example #13

Source File: Get.java From cloudExplorer with GNU General Public License v3.0

4 votes

public void run() {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    File file = new File(what);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }

    try {
        long t1 = System.currentTimeMillis();
        S3Object s3object = s3Client.getObject(new GetObjectRequest(bucket, what, version));
        InputStream objectData = s3object.getObjectContent();
        this.writeFile(objectData, destination);
        long t2 = System.currentTimeMillis();
        long diff = t2 - t1;

        if (!mainFrame.perf) {
            if (terminal) {
                System.out.print("\nDownloaded: " + what + " in " + diff / 1000 + " second(s).\n");
            } else {
                mainFrame.jTextArea1.append("\nDownloaded: " + what + " in " + diff / 1000 + " second(s).");
                mainFrame.calibrateTextArea();
            }
        }

    } catch (AmazonServiceException ase) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + ase.getMessage());
            System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
            System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
            System.out.print("\nError Type:       " + ase.getErrorType());
            System.out.print("\nRequest ID:       " + ase.getRequestId());
        }
    } catch (Exception get) {
    }
    calibrate();
}

Example #14

Source File: BucketTransition.java From cloudExplorer with GNU General Public License v3.0

4 votes

public void run() {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    int converted_days = 0;
    if (!disabled) {
        converted_days = Integer.parseInt(days);
    }
    BucketLifecycleConfiguration.Rule ruleArchiveAndExpire = null;
    if (!disabled) {
        ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule()
                .withPrefix(prefix)
                .withExpirationInDays(converted_days)
                .withStatus(BucketLifecycleConfiguration.ENABLED.toString());
    } else {
        ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule()
                .withPrefix(prefix)
                .withExpirationInDays(100)
                .withStatus(BucketLifecycleConfiguration.DISABLED.toString());
    }
    List<BucketLifecycleConfiguration.Rule> rules = new ArrayList<BucketLifecycleConfiguration.Rule>();
    rules.add(ruleArchiveAndExpire);

    try {
        BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration()
                .withRules(rules);
        s3Client.setBucketLifecycleConfiguration(bucket, configuration);
    } catch (Exception get) {
        mainFrame.jTextArea1.append("\n" + get.getMessage());
    }
    if (!disabled) {
        mainFrame.jTextArea1.append("\nSent request to change bucket life cycle to " + converted_days + " day(s). Please observe for any errors.");
    } else {
        mainFrame.jTextArea1.append("\nSent request to disable the bucket life cycle. Please observe for any errors.");
    }
    calibrate();
    mainFrame.jPanel9.setVisible(true);
}

Example #15

Source File: RestoreObject.java From cloudExplorer with GNU General Public License v3.0

4 votes

public void run() {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    File file = new File(what);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }

    try {
        RestoreObjectRequest requestRestore = new RestoreObjectRequest(bucket, what, 2);
        s3Client.restoreObject(requestRestore);

        GetObjectMetadataRequest requestCheck = new GetObjectMetadataRequest(bucket, what);
        ObjectMetadata response = s3Client.getObjectMetadata(requestCheck);

        Boolean restoreFlag = response.getOngoingRestore();
        mainFrame.jTextArea1.append("\nRestoration in progress. Please try to access the file again in a few hours.");
        calibrate();
    } catch (AmazonS3Exception amazonS3Exception) {
        mainFrame.jTextArea1.append("\nAn Amazon S3 error occurred. Exception: %s" + amazonS3Exception.toString());
        calibrate();
    } catch (Exception ex) {
        mainFrame.jTextArea1.append("\nException: %s" + ex.toString());
        calibrate();
    }

    calibrate();
}

Example #16

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String deleteBucket(String access_key, String secret_key,
        String bucket, String endpoint
) {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    message = ("\nDeleting bucket: " + bucket);
    try {
        s3Client.deleteBucket(new DeleteBucketRequest(bucket));
    } catch (Exception Delete) {
        if (terminal) {
            System.out.print("\n\n\n" + Delete.getMessage() + "\n\n\n");
        } else {
            message = message + "\n" + Delete.getMessage();
        }
    }

    if (message == null) {
        message = "\nDelete operation failed.";
    }

    return message;
}

Example #17

Source File: AmazonS3Mock.java From Scribengin with GNU Affero General Public License v3.0

4 votes

@Override
public void setS3ClientOptions(S3ClientOptions clientOptions) {
  // TODO Auto-generated method stub

}

Example #18

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String getObjectInfo(String key, String access_key,
        String secret_key, String bucket,
        String endpoint, String process
) {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    objectlist = null;

    try {
        ObjectListing current = s3Client.listObjects((bucket));

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (process.contains("checkmd5")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = objectSummary.getETag();
                        break;
                    }
                }
                if (process.contains("objectsize")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getSize());
                        break;
                    }
                }

                if (process.contains("objectdate")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getLastModified());
                        break;
                    }

                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (Exception listBucket) {
        //  mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
    }

    return objectlist;
}

Example #19

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String listBuckets(String access_key, String secret_key, String endpoint) {

        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration());
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
        String[] array = new String[10];

        String bucketlist = null;

        int i = 0;
        try {

            for (Bucket bucket : s3Client.listBuckets()) {
                bucketlist = bucketlist + " " + bucket.getName();
            }

        } catch (AmazonServiceException ase) {
            if (NewJFrame.gui) {
                mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
                mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
                mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
                mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
                mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
                calibrate();
            } else {
                System.out.print("\n\nError Message:    " + ase.getMessage());
                System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
                System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
                System.out.print("\nError Type:       " + ase.getErrorType());
                System.out.print("\nRequest ID:       " + ase.getRequestId());
            }

        } catch (Exception lsbuckets) {

            if (lsbuckets.getMessage().contains("peer not authenticated") || lsbuckets.getMessage().contains("hostname in certificate didn't match")) {
                if (NewJFrame.gui) {
                    mainFrame.jTextArea1.append("\nError: This program does not support non-trusted SSL certificates\n\nor your SSL certificates are incorrect.");
                } else {
                    System.out.print("\n\nError: This program does not support non-trusted SSL certificates\n\nor your SSL certificates are not correctly installed.");
                }
            }
        }
        String parse = null;

        if (bucketlist != null) {
            parse = bucketlist.replace("null", "");

        } else {
            parse = "no_bucket_found";
        }

        return parse;
    }

Example #20

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String abortMPUploads(String access_key, String secret_key, String bucket, String endpoint) {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());

    try {
        if (endpoint.contains("amazonaws.com")) {
            String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
            if (aws_endpoint.contains("US")) {
                s3Client.setEndpoint("https://s3.amazonaws.com");
            } else if (aws_endpoint.contains("us-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("eu-west")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("ap-")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else if (aws_endpoint.contains("sa-east-1")) {
                s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
            } else {
                s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
            }
        } else {
            s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
            s3Client.setEndpoint(endpoint);
        }
        TransferManager tm = new TransferManager(s3Client);
        int month = 1000 * 60 * 60 * 24 * 30;
        Date oneWeekAgo = new Date(System.currentTimeMillis() - month);
        tm.abortMultipartUploads(bucket, oneWeekAgo);
        message = ("\nSent request to delete all the multi-part uploads in the past month");
    } catch (AmazonServiceException multipart) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + multipart.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + multipart.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + multipart.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + multipart.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + multipart.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + multipart.getMessage());
            System.out.print("\nHTTP Status Code: " + multipart.getStatusCode());
            System.out.print("\nAWS Error Code:   " + multipart.getErrorCode());
            System.out.print("\nError Type:       " + multipart.getErrorType());
            System.out.print("\nRequest ID:       " + multipart.getRequestId());
        }
    }
    if (message == null) {
        message = "Failed to list multi-part uploads.";
    }
    return message;

}

Example #21

Source File: BucketClass.java From cloudExplorer with GNU General Public License v3.0

4 votes

String makeBucket(String access_key, String secret_key, String bucket, String endpoint, String region) {
    String message = null;
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());

    if (endpoint.contains("amazonaws.com")) {
        s3Client.setEndpoint(endpoint);

        if (region.length() > 5) {
            if (region.contains("us-east-1")) {
                s3Client.setEndpoint("https://s3.amazonaws.com");
            } else if (region.contains("us-west")) {
                s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com");
            } else if (region.contains("eu-west")) {
                s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com");
            } else if (region.contains("ap-")) {
                s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com");
            } else if (region.contains("sa-east-1")) {
                s3Client.setEndpoint("https://s3-" + region + ".amazonaws.com");
            } else {
                s3Client.setEndpoint("https://s3." + region + ".amazonaws.com");
            }
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }

    message = ("\nAttempting to create the bucket. Please view the Bucket list window for an update.");

    try {
        if (!(s3Client.doesBucketExist(bucket))) {
            s3Client.createBucket(new CreateBucketRequest(bucket));
        }
        String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
    } catch (AmazonServiceException ase) {
        if (ase.getMessage().contains("InvalidLocationConstraint")) {
            s3Client.createBucket(new CreateBucketRequest(bucket, region));
        } else {
            if (NewJFrame.gui) {
                mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
                mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
                mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
                mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
                mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
                calibrate();
            } else {
                System.out.print("\n\nError Message:    " + ase.getMessage());
                System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
                System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
                System.out.print("\nError Type:       " + ase.getErrorType());
                System.out.print("\nRequest ID:       " + ase.getRequestId());
            }
        }
    }
    return message;
}

Example #22

Source File: Delete.java From cloudExplorer with GNU General Public License v3.0

4 votes

public void run() {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }

    try {

        if (version != null) {
            s3Client.deleteVersion(new DeleteVersionRequest(bucket, what, version));
        } else {
            s3Client.deleteObject(new DeleteObjectRequest(bucket, what));
        }
        if (!debug) {
            NewJFrame.jTextArea1.append("\nDeleted file: " + what);
        }
        calibrate();
    } catch (AmazonServiceException ase) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + ase.getMessage());
            System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
            System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
            System.out.print("\nError Type:       " + ase.getErrorType());
            System.out.print("\nRequest ID:       " + ase.getRequestId());
        }
    } catch (Exception delete) {
    }
}

Example #23

Source File: BucketTransitionGlacier.java From cloudExplorer with GNU General Public License v3.0

4 votes

public void run() {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    int converted_days = 0;
    if (!disabled) {
        converted_days = Integer.parseInt(days);
    }

    Transition transToArchive = new Transition()
            .withDays(converted_days)
            .withStorageClass(StorageClass.Glacier);

    BucketLifecycleConfiguration.Rule ruleArchiveAndExpire = null;
    if (!disabled) {
        ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule()
                .withPrefix(prefix)
                .withTransition(transToArchive)
                // .withExpirationInDays(converted_days + 1)
                .withStatus(BucketLifecycleConfiguration.ENABLED.toString());
    } else {
        ruleArchiveAndExpire = new BucketLifecycleConfiguration.Rule()
                .withPrefix(prefix)
                .withTransition(transToArchive)
                //.withExpirationInDays(100)
                .withStatus(BucketLifecycleConfiguration.DISABLED.toString());
    }
    List<BucketLifecycleConfiguration.Rule> rules = new ArrayList<BucketLifecycleConfiguration.Rule>();
    rules.add(ruleArchiveAndExpire);

    try {
        BucketLifecycleConfiguration configuration = new BucketLifecycleConfiguration()
                .withRules(rules);
        s3Client.setBucketLifecycleConfiguration(bucket, configuration);
    } catch (Exception get) {
        mainFrame.jTextArea1.append("\n" + get.getMessage());
    }
    if (!disabled) {
        mainFrame.jTextArea1.append("\nSent request to set bucket life cycle to tier to Glacier after: " + converted_days + " day(s). Please observe for any errors.");
    } else {
        mainFrame.jTextArea1.append("\nSent request to disable the bucket life cycle. Please observe for any errors.");
    }
    calibrate();
}

Example #24

Source File: S3PersistReader.java From streams with Apache License 2.0

4 votes

@Override
public void prepare(Object configurationObject) {

  streamsConfiguration = StreamsConfigurator.detectConfiguration();

  lineReaderUtil = LineReadWriteUtil.getInstance(s3ReaderConfiguration);

  // Connect to S3
  synchronized (this) {
    // Create the credentials Object
    AWSCredentials credentials = new BasicAWSCredentials(s3ReaderConfiguration.getKey(), s3ReaderConfiguration.getSecretKey());

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toString()));

    // We do not want path style access
    S3ClientOptions clientOptions = new S3ClientOptions();
    clientOptions.setPathStyleAccess(false);

    this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
    if (StringUtils.isNotEmpty(s3ReaderConfiguration.getRegion())) {
      this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion())));
    }
    this.amazonS3Client.setS3ClientOptions(clientOptions);
  }

  final ListObjectsRequest request = new ListObjectsRequest()
      .withBucketName(this.s3ReaderConfiguration.getBucket())
      .withPrefix(s3ReaderConfiguration.getReaderPath())
      .withMaxKeys(500);


  ObjectListing listing = this.amazonS3Client.listObjects(request);

  this.files = new ArrayList<>();

  /*
   * If you can list files that are in this path, then you must be dealing with a directory
   * if you cannot list files that are in this path, then you are most likely dealing with
   * a simple file.
   */
  boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0;
  boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0;

  if (hasCommonPrefixes || hasObjectSummaries) {
    // Handle the 'directory' use case
    do {
      if (hasCommonPrefixes) {
        for (String file : listing.getCommonPrefixes()) {
          this.files.add(file);
        }
      } else {
        for (final S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
          this.files.add(objectSummary.getKey());
        }
      }

      // get the next batch.
      listing = this.amazonS3Client.listNextBatchOfObjects(listing);
    }
    while (listing.isTruncated());
  } else {
    // handle the single file use-case
    this.files.add(s3ReaderConfiguration.getReaderPath());
  }

  if (this.files.size() <= 0) {
    LOGGER.error("There are no files to read");
  }

  this.persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(streamsConfiguration.getQueueSize().intValue()));
  this.executor = Executors.newSingleThreadExecutor();
}

Example #25

Source File: AmazonS3Stub.java From aws-java-sdk-stubs with Apache License 2.0

4 votes

@Override
public void setS3ClientOptions(final S3ClientOptions arg0) {
  throw new UnsupportedOperationException();
}

Example #26

Source File: MockAmazonS3.java From tajo with Apache License 2.0

4 votes

@Override
public void setS3ClientOptions(S3ClientOptions clientOptions) {
  throw new TajoInternalError(new UnsupportedException());
}

Example #27

Source File: DummyS3Client.java From ignite with Apache License 2.0

4 votes

/** Unsupported Operation. */
@Override public void setS3ClientOptions(S3ClientOptions clientOptions) {
    throw new UnsupportedOperationException("Operation not supported");
}

Example #28

Source File: S3WaitOperatorFactoryTest.java From digdag with Apache License 2.0

4 votes

@Test
public void testDefaults()
        throws Exception
{
    Config config = newConfig();

    config.set("_command", BUCKET + "/" + KEY);

    when(taskRequest.getConfig()).thenReturn(config);

    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentType(CONTENT_TYPE);
    objectMetadata.setContentLength(CONTENT_LENGTH);
    objectMetadata.setUserMetadata(USER_METADATA);

    Operator operator = factory.newOperator(newContext(projectPath, taskRequest));

    when(s3Client.getObjectMetadata(objectMetadataRequestCaptor.capture())).thenReturn(objectMetadata);

    TaskResult taskResult = operator.run();

    verify(s3ClientFactory).create(credentialsCaptor.capture(), clientConfigurationCaptor.capture());

    ClientConfiguration clientConfiguration = clientConfigurationCaptor.getValue();
    assertThat(clientConfiguration.getProxyHost(), is(nullValue()));
    assertThat(clientConfiguration.getProxyPort(), is(-1));
    assertThat(clientConfiguration.getProxyUsername(), is(nullValue()));
    assertThat(clientConfiguration.getProxyPassword(), is(nullValue()));

    verify(s3Client).setS3ClientOptions(s3ClientOptionsCaptor.capture());
    S3ClientOptions s3ClientOptions = s3ClientOptionsCaptor.getValue();
    assertThat(s3ClientOptions.isPathStyleAccess(), is(false));

    AWSCredentials credentials = credentialsCaptor.getValue();
    assertThat(credentials.getAWSAccessKeyId(), is(ACCESS_KEY_ID));
    assertThat(credentials.getAWSSecretKey(), is(SECRET_ACCESS_KEY));

    GetObjectMetadataRequest objectMetadataRequest = objectMetadataRequestCaptor.getValue();
    assertThat(objectMetadataRequest.getKey(), is(KEY));
    assertThat(objectMetadataRequest.getBucketName(), is(BUCKET));
    assertThat(objectMetadataRequest.getSSECustomerKey(), is(nullValue()));

    Config expectedStoreParams = newConfig();
    expectedStoreParams
            .getNestedOrSetEmpty("s3")
            .getNestedOrSetEmpty("last_object")
            .set("metadata", objectMetadata.getRawMetadata())
            .set("user_metadata", objectMetadata.getUserMetadata());

    assertThat(taskResult.getStoreParams(), is(expectedStoreParams));
}