Java Code Examples for com.amazonaws.services.s3.AmazonS3Client#setEndpoint()

private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException {
    final String accessKey = ctx.getAccountId();
    final String secretKey = ctx.getSecretKey();
    String endpoint = getEndpoint(ctx);
    LOGGER.info("endpoint: {}", endpoint);

    final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey);
    final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials);
    amazonS3Client.setEndpoint(endpoint);

    if (ctx.usesEmc()) {
        final S3ClientOptions options = new S3ClientOptions();
        options.setPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(options);
    }

    return amazonS3Client;
}
 

@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    projectDir = folder.getRoot().toPath().resolve("foobar");
    config = folder.newFile().toPath();

    client = DigdagClient.builder()
            .host(server.host())
            .port(server.port())
            .build();

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);

    s3.createBucket(archiveBucket);
    s3.createBucket(logStorageBucket);
}
 

@Override
public Storage newStorage(Config config)
{
    AmazonS3Client client = new AmazonS3Client(
            buildCredentialsProvider(config),
            buildClientConfiguration(config));
    if (config.has("endpoint")) {
        client.setEndpoint(config.get("endpoint", String.class));
    }

    if (config.has("path-style-access")) {
        client.setS3ClientOptions(
          S3ClientOptions.builder().setPathStyleAccess(
            config.get("path-style-access", Boolean.class, false)
          ).build());
    }

    String bucket = config.get("bucket", String.class);

    return new S3Storage(config, client, bucket);
}
 

@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    AmazonS3Client s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);

    String bucket = UUID.randomUUID().toString();
    s3.createBucket(bucket);

    ConfigFactory cf = new ConfigFactory(objectMapper());
    Config config = cf.create()
        .set("endpoint", TEST_S3_ENDPOINT)
        .set("bucket", bucket)  // use unique bucket name
        .set("credentials.access-key-id", TEST_S3_ACCESS_KEY_ID)
        .set("credentials.secret-access-key", TEST_S3_SECRET_ACCESS_KEY)
        ;
    storage = new S3StorageFactory().newStorage(config);
}
 

/**
 * Return an AmazonS3Client set up with the proper endpoint
 * defined in core-site.xml using a property like fs.s3a.endpoint.
 * This mimics code found in S3AFileSystem.
 *
 * @param conf
 * @param scheme
 * @return
 */
private static AmazonS3Client getS3Client(Configuration conf, String scheme)
{
  AmazonS3Client s3Client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
  String endpointKey = "fs." + scheme.toLowerCase() + ".endpoint";
  String endPoint = conf.getTrimmed(endpointKey,"");
  log.debug("Using endpoint setting " + endpointKey);
  if (!endPoint.isEmpty()) {
    try {
      log.debug("Setting S3 client endpoint to " + endPoint);
      s3Client.setEndpoint(endPoint);
    } catch (IllegalArgumentException e) {
      String msg = "Incorrect endpoint: "  + e.getMessage();
      log.error(msg);
      throw new IllegalArgumentException(msg, e);
    }
  }
  return s3Client;
}
 

/** Returns an S3 client given the configuration **/
public static AmazonS3Client getS3Client(Map conf) {
    AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
    AWSCredentials credentials = provider.getCredentials();
    ClientConfiguration config = new ClientConfiguration();

    AmazonS3Client client = new AmazonS3Client(credentials, config);

    String regionName = ConfUtils.getString(conf, REGION);
    if (StringUtils.isNotBlank(regionName)) {
        client.setRegion(RegionUtils.getRegion(regionName));
    }

    String endpoint = ConfUtils.getString(conf, ENDPOINT);
    if (StringUtils.isNotBlank(endpoint)) {
        client.setEndpoint(endpoint);
    }
    return client;
}
 

public RedshiftManifestEmitter(KinesisConnectorConfiguration configuration) {
    dataTable = configuration.REDSHIFT_DATA_TABLE;
    fileTable = configuration.REDSHIFT_FILE_TABLE;
    fileKeyColumn = configuration.REDSHIFT_FILE_KEY_COLUMN;
    dataDelimiter = configuration.REDSHIFT_DATA_DELIMITER;
    copyMandatory = configuration.REDSHIFT_COPY_MANDATORY;
    s3Bucket = configuration.S3_BUCKET;
    s3Endpoint = configuration.S3_ENDPOINT;
    s3Client = new AmazonS3Client(configuration.AWS_CREDENTIALS_PROVIDER);
    if (s3Endpoint != null) {
        s3Client.setEndpoint(s3Endpoint);
    }
    credentialsProvider = configuration.AWS_CREDENTIALS_PROVIDER;
    loginProps = new Properties();
    loginProps.setProperty("user", configuration.REDSHIFT_USERNAME);
    loginProps.setProperty("password", configuration.REDSHIFT_PASSWORD);
    redshiftURL = configuration.REDSHIFT_URL;
}
 

@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    proxyServer = TestUtils.startRequestFailingProxy(10);

    server = TemporaryDigdagServer.builder()
            .environment(ImmutableMap.of(
                    "http_proxy", "http://" + proxyServer.getListenAddress().getHostString() + ":" + proxyServer.getListenAddress().getPort())
            )
            .configuration(
                    "digdag.secret-encryption-key = " + Base64.getEncoder().encodeToString(RandomUtils.nextBytes(16)))
            .build();

    server.start();

    projectDir = folder.getRoot().toPath().resolve("foobar");

    client = DigdagClient.builder()
            .host(server.host())
            .port(server.port())
            .build();

    bucket = UUID.randomUUID().toString();

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);
    s3.createBucket(bucket);
}
 

/**
 * Initialize the AmazonS3 client using the accessKey, secretAccessKey, sets
 * endpoint for the s3Client if provided
 *
 * @param accessKey
 * @param secretAccessKey
 * @param endPoint
 */
public void initializeS3Client(@javax.validation.constraints.NotNull String accessKey,
    @javax.validation.constraints.NotNull String secretAccessKey, String endPoint)
{
  Preconditions.checkNotNull(accessKey);
  Preconditions.checkNotNull(secretAccessKey);
  s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
  if (endPoint != null) {
    s3Client.setEndpoint(endPoint);
  }
}
 

/**
 * Instantiates {@code AmazonS3Client} instance.
 *
 * @return Client instance to use to connect to AWS.
 */
AmazonS3Client createAmazonS3Client() {
    AmazonS3Client cln = cfg != null
        ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg))
        : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider));

    if (!F.isEmpty(bucketEndpoint))
        cln.setEndpoint(bucketEndpoint);

    return cln;
}
 

public static AmazonS3 create(
    AWSCredentialsProvider awsCredentialsProvider, S3Properties s3Properties) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  if (s3Properties.getProxyProtocol() != null) {
    if (s3Properties.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
      clientConfiguration.setProtocol(Protocol.HTTPS);
    } else {
      clientConfiguration.setProtocol(Protocol.HTTP);
    }
    Optional.ofNullable(s3Properties.getProxyHost()).ifPresent(clientConfiguration::setProxyHost);
    Optional.ofNullable(s3Properties.getProxyPort())
        .map(Integer::parseInt)
        .ifPresent(clientConfiguration::setProxyPort);
  }

  AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

  if (!StringUtils.isEmpty(s3Properties.getEndpoint())) {
    client.setEndpoint(s3Properties.getEndpoint());

    if (!StringUtils.isEmpty(s3Properties.getRegionOverride())) {
      client.setSignerRegionOverride(s3Properties.getRegionOverride());
    }

    client.setS3ClientOptions(
        S3ClientOptions.builder().setPathStyleAccess(s3Properties.getPathStyleAccess()).build());
  } else {
    Optional.ofNullable(s3Properties.getRegion())
        .map(Regions::fromName)
        .map(Region::getRegion)
        .ifPresent(client::setRegion);
  }

  return client;
}
 

public S3Emitter(KinesisConnectorConfiguration configuration) {
    s3Bucket = configuration.S3_BUCKET;
    s3Endpoint = configuration.S3_ENDPOINT;
    s3client = new AmazonS3Client(configuration.AWS_CREDENTIALS_PROVIDER);
    if (s3Endpoint != null) {
        s3client.setEndpoint(s3Endpoint);
    }
}
 

/**
 * Helper method to create the Amazon S3 bucket.
 * 
 * @param s3Bucket
 *        The name of the bucket to create
 */
private void createS3Bucket(String s3Bucket) {
    AmazonS3Client client = new AmazonS3Client(config.AWS_CREDENTIALS_PROVIDER);
    client.setEndpoint(config.S3_ENDPOINT);
    LOG.info("Creating Amazon S3 bucket " + s3Bucket);
    S3Utils.createBucket(client, s3Bucket);
}
 

public AmazonS3Client getS3Client() throws InvalidInputException {
    AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, getClientConfiguration());
    client.setEndpoint("https://s3." + region + getAwsClientSuffix(region));
    return client;
}
 

public BeanstalkDeployer(String s3Endpoint, String beanstalkEndpoint, AWSCredentialsProvider credentialsProvider) {
    s3 = new AmazonS3Client(credentialsProvider);
    elasticBeanstalk = new AWSElasticBeanstalkClient(credentialsProvider);
    s3.setEndpoint(s3Endpoint);
    elasticBeanstalk.setEndpoint(beanstalkEndpoint);
}
 

public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }