Java Code Examples for com.amazonaws.ClientConfiguration#setSocketTimeout()

The following examples show how to use com.amazonaws.ClientConfiguration#setSocketTimeout() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3Service.java    From crate with Apache License 2.0 6 votes vote down vote up
static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) {
    final ClientConfiguration clientConfiguration = new ClientConfiguration();
    // the response metadata cache is only there for diagnostics purposes,
    // but can force objects from every response to the old generation.
    clientConfiguration.setResponseMetadataCacheSize(0);
    clientConfiguration.setProtocol(clientSettings.protocol);

    if (Strings.hasText(clientSettings.proxyHost)) {
        // TODO: remove this leniency, these settings should exist together and be validated
        clientConfiguration.setProxyHost(clientSettings.proxyHost);
        clientConfiguration.setProxyPort(clientSettings.proxyPort);
        clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
        clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
    }

    clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries);
    clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries);
    clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);

    return clientConfiguration;
}
 
Example 2
Source File: S3Accessor.java    From datacollector with Apache License 2.0 6 votes vote down vote up
ClientConfiguration createClientConfiguration() throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(connectionConfigs.getConnectionTimeoutMillis());
  clientConfig.setSocketTimeout(connectionConfigs.getSocketTimeoutMillis());
  clientConfig.withMaxErrorRetry(connectionConfigs.getMaxErrorRetry());

  if (connectionConfigs.isProxyEnabled()) {
    clientConfig.setProxyHost(connectionConfigs.getProxyHost());
    clientConfig.setProxyPort(connectionConfigs.getProxyPort());
    if (connectionConfigs.isProxyAuthenticationEnabled()) {
      clientConfig.setProxyUsername(connectionConfigs.getProxyUser().get());
      clientConfig.setProxyPassword(connectionConfigs.getProxyPassword().get());
    }
  }
  return clientConfig;
}
 
Example 3
Source File: AWSUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up
public static ClientConfiguration getClientConfiguration(ProxyConfig config) throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(config.connectionTimeout * MILLIS);
  clientConfig.setSocketTimeout(config.socketTimeout * MILLIS);
  clientConfig.withMaxErrorRetry(config.retryCount);

  // Optional proxy settings
  if (config.useProxy) {
    if (config.proxyHost != null && !config.proxyHost.isEmpty()) {
      clientConfig.setProxyHost(config.proxyHost);
      clientConfig.setProxyPort(config.proxyPort);

      if (config.proxyUser != null && !config.proxyUser.get().isEmpty()) {
        clientConfig.setProxyUsername(config.proxyUser.get());
      }

      if (config.proxyPassword != null) {
        clientConfig.setProxyPassword(config.proxyPassword.get());
      }
    }
  }
  return clientConfig;
}
 
Example 4
Source File: AwsEc2ServiceImpl.java    From crate with Apache License 2.0 5 votes vote down vote up
static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings clientSettings) {
    final ClientConfiguration clientConfiguration = new ClientConfiguration();
    // the response metadata cache is only there for diagnostics purposes,
    // but can force objects from every response to the old generation.
    clientConfiguration.setResponseMetadataCacheSize(0);
    clientConfiguration.setProtocol(clientSettings.protocol);
    if (Strings.hasText(clientSettings.proxyHost)) {
        // TODO: remove this leniency, these settings should exist together and be validated
        clientConfiguration.setProxyHost(clientSettings.proxyHost);
        clientConfiguration.setProxyPort(clientSettings.proxyPort);
        clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
        clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
    }
    // Increase the number of retries in case of 5xx API responses
    final Random rand = Randomness.get();
    final RetryPolicy retryPolicy = new RetryPolicy(
        RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
        (originalRequest, exception, retriesAttempted) -> {
            // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
            logger.warn("EC2 API request failed, retry again. Reason was:", exception);
            return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
        },
        10,
        false);
    clientConfiguration.setRetryPolicy(retryPolicy);
    clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);
    return clientConfiguration;
}
 
Example 5
Source File: AbstractAWSProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    if (sslContextService != null) {
        final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.NONE);
        SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, null);
        config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
    }

    if (context.getProperty(PROXY_HOST).isSet()) {
        String proxyHost = context.getProperty(PROXY_HOST).getValue();
        config.setProxyHost(proxyHost);
        Integer proxyPort = context.getProperty(PROXY_HOST_PORT).asInteger();
        config.setProxyPort(proxyPort);
    }

    return config;
}
 
Example 6
Source File: AWSCloudTrailProcessingExecutor.java    From aws-cloudtrail-processing-library with Apache License 2.0 5 votes vote down vote up
private void buildS3Client() {
    // override default timeout for S3Client
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setConnectionTimeout(SDK_TIME_OUT);
    clientConfiguration.setSocketTimeout(SDK_TIME_OUT);

    if (s3Client == null) {
        s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(config.getAwsCredentialsProvider())
                .withClientConfiguration(clientConfiguration)
                .withRegion(config.getS3Region())
                .build();
    }
}
 
Example 7
Source File: AWSUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public static ClientConfiguration getClientConfiguration(ProxyConfig config) throws StageException {
  ClientConfiguration clientConfig = new ClientConfiguration();

  clientConfig.setConnectionTimeout(config.connectionTimeout * MILLIS);
  clientConfig.setSocketTimeout(config.socketTimeout * MILLIS);
  clientConfig.withMaxErrorRetry(config.retryCount);

  // Optional proxy settings
  if (config.useProxy) {
    if (config.proxyHost != null && !config.proxyHost.isEmpty()) {
      clientConfig.setProxyHost(config.proxyHost);
      clientConfig.setProxyPort(config.proxyPort);

      if (config.proxyUser != null && !config.proxyUser.get().isEmpty()) {
        clientConfig.setProxyUsername(config.proxyUser.get());
      }

      if (config.proxyPassword != null && !config.proxyPassword.get().isEmpty()) {
        clientConfig.setProxyPassword(config.proxyPassword.get());
      }

      if (config.proxyDomain != null && !config.proxyDomain.isEmpty()) {
        clientConfig.setProxyDomain(config.proxyDomain);
      }

      if (config.proxyWorkstation != null && !config.proxyWorkstation.isEmpty()) {
        clientConfig.setProxyWorkstation(config.proxyWorkstation);
      }
    }
  }
  return clientConfig;
}
 
Example 8
Source File: COSAPIClient.java    From stocator with Apache License 2.0 5 votes vote down vote up
/**
 * Initializes connection management
 *
 * @param conf Hadoop configuration
 * @param clientConf client SDK configuration
 */
private void initConnectionSettings(Configuration conf,
    ClientConfiguration clientConf) throws IOException {
  clientConf.setMaxConnections(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
  clientConf.setClientExecutionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      CLIENT_EXEC_TIMEOUT, DEFAULT_CLIENT_EXEC_TIMEOUT));
  clientConf.setMaxErrorRetry(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
  clientConf.setConnectionTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
  clientConf.setSocketTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));
  clientConf.setRequestTimeout(Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT));
  int sockSendBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_SEND_BUFFER, DEFAULT_SOCKET_SEND_BUFFER);
  int sockRecvBuffer = Utils.getInt(conf, FS_COS, FS_ALT_KEYS,
      SOCKET_RECV_BUFFER, DEFAULT_SOCKET_RECV_BUFFER);
  clientConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
  String signerOverride = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      SIGNING_ALGORITHM, "");
  if (!signerOverride.isEmpty()) {
    LOG.debug("Signer override = {}", signerOverride);
    clientConf.setSignerOverride(signerOverride);
  }

  String userAgentPrefix = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS,
      USER_AGENT_PREFIX, DEFAULT_USER_AGENT_PREFIX);
  String userAgentName = singletoneInitTimeData.getUserAgentName();
  if (!userAgentPrefix.equals(DEFAULT_USER_AGENT_PREFIX)) {
    userAgentName = userAgentPrefix + " " + userAgentName;
  }
  clientConf.setUserAgentPrefix(userAgentName);
}
 
Example 9
Source File: AwsModuleTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testAwsHttpClientConfigurationSerializationDeserialization() throws Exception {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setConnectionTimeout(100);
  clientConfiguration.setConnectionMaxIdleMillis(1000);
  clientConfiguration.setSocketTimeout(300);

  final String valueAsJson = objectMapper.writeValueAsString(clientConfiguration);
  final ClientConfiguration clientConfigurationDeserialized =
      objectMapper.readValue(valueAsJson, ClientConfiguration.class);
  assertEquals(100, clientConfigurationDeserialized.getConnectionTimeout());
  assertEquals(1000, clientConfigurationDeserialized.getConnectionMaxIdleMillis());
  assertEquals(300, clientConfigurationDeserialized.getSocketTimeout());
}
 
Example 10
Source File: ProducerUtils.java    From kinesis-aggregation with Apache License 2.0 5 votes vote down vote up
/**
 * Create a new Kinesis producer for publishing to Kinesis.
 * 
 * @param region The region of the Kinesis stream to publish to.
 * 
 * @return An Amazon Kinesis producer for publishing to a Kinesis stream.
 */
public static AmazonKinesis getKinesisProducer(String region)
{
    ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(25);
    config.setConnectionTimeout(60000);
    config.setSocketTimeout(60000);

    AmazonKinesis producer = new AmazonKinesisClient(new DefaultAWSCredentialsProviderChain(), config);
    producer.setRegion(Region.getRegion(Regions.fromName(region)));

    return producer;
}
 
Example 11
Source File: LambdaAggregatingForwarder.java    From kinesis-aggregation with Apache License 2.0 5 votes vote down vote up
/**
 * One-time initialization of resources for this Lambda function.
 */
public LambdaAggregatingForwarder()
{
    this.aggregator = new RecordAggregator();
    
    /*
     * If the Kinesis stream you're forwarding to is in the same account as this AWS Lambda function, you can just give the IAM Role executing
     * this function permissions to publish to the stream and DefaultAWSCredentialsProviderChain() will take care of it.  
     * 
     * If you're publishing to a Kinesis stream in another AWS account, it's trickier.  Kinesis doesn't currently provide cross-account publishing
     * permissions. You must create an IAM role in the AWS account with the DESTINATION stream that has permissions to publish to that stream
     * (fill that IAM role's ARN in for "<RoleToAssumeARN>" below) and then the IAM role executing this Lambda function must be given permission
     * to assume the role "<RoleToAssumeARN>" from the other AWS account.  
     */
    AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
    //AWSCredentialsProvider provider = new STSAssumeRoleSessionCredentialsProvider(new DefaultAWSCredentialsProviderChain(), "<RoleToAssumeARN>", "KinesisForwarder");
    
    //Set max conns to 1 since we use this client serially
    ClientConfiguration kinesisConfig = new ClientConfiguration();
    kinesisConfig.setMaxConnections(1);
    kinesisConfig.setProtocol(Protocol.HTTPS);
    kinesisConfig.setConnectionTimeout(DESTINATION_CONNECTION_TIMEOUT);
    kinesisConfig.setSocketTimeout(DESTINATION_SOCKET_TIMEOUT);
    
    this.kinesisForwarder = new AmazonKinesisClient(provider, kinesisConfig);
    this.kinesisForwarder.setRegion(Region.getRegion(DESTINATION_STREAM_REGION));
}
 
Example 12
Source File: generateDataset.java    From ecosys with Apache License 2.0 5 votes vote down vote up
public void run(String target_host, String target_path, String rest_endpoint, int times) {
  // create spark context and spark sql instance
  SparkSession sc = SparkSession.builder().appName("Uber").getOrCreate();
  sc.conf().set("fs.s3a.attempts.maximum", "30");
  sc.conf().set("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
  // conf.set("mapreduce.fileoutputcommitter.algorithm.version", "2")

  // set the S3 client configuration and create a client connected to S3.
  ClientConfiguration configuration = new ClientConfiguration();
  configuration.setMaxErrorRetry(10);
  configuration.setConnectionTimeout(501000);
  configuration.setSocketTimeout(501000);
  //configuration.setUseTcpKeepAlive(true);
  AmazonS3Client s3client = new AmazonS3Client(configuration);

  if (!awsAccessKey.trim().equals("") && !awsSecretKey.trim().equals("")) {
    logInfo("Read S3 credentials", true);
    s3client = new AmazonS3Client(new BasicAWSCredentials(awsAccessKey, awsSecretKey), configuration);
    sc.conf().set("fs.s3a.access.key", awsAccessKey);
    sc.conf().set("fs.s3a.secret.key", awsSecretKey);
  }

  logInfo("Start process", true);
  for (String schema_folder : schema_folders) {
    logInfo("Start to load schema folder " + schema_folder, true);
    post2Tg(s3client, sc, schema_folder, target_host, target_path, rest_endpoint, times);
    logInfo("Loding schema folder " + schema_folder + " successfully", true);
  }
  logInfo("End process", true);
}
 
Example 13
Source File: AwsModule.java    From beam with Apache License 2.0 4 votes vote down vote up
@Override
public ClientConfiguration deserialize(JsonParser jsonParser, DeserializationContext context)
    throws IOException {
  Map<String, Object> map = jsonParser.readValueAs(new TypeReference<Map<String, Object>>() {});

  ClientConfiguration clientConfiguration = new ClientConfiguration();

  if (map.containsKey(PROXY_HOST)) {
    clientConfiguration.setProxyHost((String) map.get(PROXY_HOST));
  }
  if (map.containsKey(PROXY_PORT)) {
    clientConfiguration.setProxyPort((Integer) map.get(PROXY_PORT));
  }
  if (map.containsKey(PROXY_USERNAME)) {
    clientConfiguration.setProxyUsername((String) map.get(PROXY_USERNAME));
  }
  if (map.containsKey(PROXY_PASSWORD)) {
    clientConfiguration.setProxyPassword((String) map.get(PROXY_PASSWORD));
  }
  if (map.containsKey(CLIENT_EXECUTION_TIMEOUT)) {
    clientConfiguration.setClientExecutionTimeout((Integer) map.get(CLIENT_EXECUTION_TIMEOUT));
  }
  if (map.containsKey(CONNECTION_MAX_IDLE_TIME)) {
    clientConfiguration.setConnectionMaxIdleMillis(
        ((Number) map.get(CONNECTION_MAX_IDLE_TIME)).longValue());
  }
  if (map.containsKey(CONNECTION_TIMEOUT)) {
    clientConfiguration.setConnectionTimeout((Integer) map.get(CONNECTION_TIMEOUT));
  }
  if (map.containsKey(CONNECTION_TIME_TO_LIVE)) {
    clientConfiguration.setConnectionTTL(
        ((Number) map.get(CONNECTION_TIME_TO_LIVE)).longValue());
  }
  if (map.containsKey(MAX_CONNECTIONS)) {
    clientConfiguration.setMaxConnections((Integer) map.get(MAX_CONNECTIONS));
  }
  if (map.containsKey(REQUEST_TIMEOUT)) {
    clientConfiguration.setRequestTimeout((Integer) map.get(REQUEST_TIMEOUT));
  }
  if (map.containsKey(SOCKET_TIMEOUT)) {
    clientConfiguration.setSocketTimeout((Integer) map.get(SOCKET_TIMEOUT));
  }
  return clientConfiguration;
}
 
Example 14
Source File: AwsS3Storage.java    From ecs-sync with Apache License 2.0 4 votes vote down vote up
@Override
public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) {
    super.configure(source, filters, target);

    Assert.hasText(config.getAccessKey(), "accessKey is required");
    Assert.hasText(config.getSecretKey(), "secretKey is required");
    Assert.hasText(config.getBucketName(), "bucketName is required");
    Assert.isTrue(config.getBucketName().matches("[A-Za-z0-9._-]+"), config.getBucketName() + " is not a valid bucket name");

    AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    ClientConfiguration cc = new ClientConfiguration();

    if (config.getProtocol() != null)
        cc.setProtocol(Protocol.valueOf(config.getProtocol().toString().toUpperCase()));

    if (config.isLegacySignatures()) cc.setSignerOverride("S3SignerType");

    if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs());

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(creds))
            .withClientConfiguration(cc);

    if (config.getHost() != null) {
        String endpoint = "";
        if (config.getProtocol() != null) endpoint += config.getProtocol() + "://";
        endpoint += config.getHost();
        if (config.getPort() > 0) endpoint += ":" + config.getPort();
        builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegion()));
    } else if (config.getRegion() != null) {
        builder.withRegion(config.getRegion());
    }

    if (config.isDisableVHosts()) {
        log.info("The use of virtual hosted buckets has been DISABLED.  Path style buckets will be used.");
        builder.withPathStyleAccessEnabled(true);
    }

    s3 = builder.build();

    boolean bucketExists = s3.doesBucketExistV2(config.getBucketName());

    boolean bucketHasVersions = false;
    if (bucketExists && config.isIncludeVersions()) {
        // check if versioning has ever been enabled on the bucket (versions will not be collected unless required)
        BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(config.getBucketName());
        List<String> versionedStates = Arrays.asList(BucketVersioningConfiguration.ENABLED, BucketVersioningConfiguration.SUSPENDED);
        bucketHasVersions = versionedStates.contains(versioningConfig.getStatus());
    }

    if (config.getKeyPrefix() == null) config.setKeyPrefix(""); // make sure keyPrefix isn't null

    if (source == this) {
        if (config.getExcludedKeys() != null) {
            excludedKeyPatterns = new ArrayList<>();
            for (String pattern : config.getExcludedKeys()) {
                excludedKeyPatterns.add(Pattern.compile(pattern));
            }
        }
    }

    if (target == this) {
        // create bucket if it doesn't exist
        if (!bucketExists && config.isCreateBucket()) {
            s3.createBucket(config.getBucketName());
            bucketExists = true;
            if (config.isIncludeVersions()) {
                s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(config.getBucketName(),
                        new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
                bucketHasVersions = true;
            }
        }

        // make sure MPU settings are valid
        if (config.getMpuThresholdMb() > MAX_PUT_SIZE_MB) {
            log.warn("{}MB is above the maximum PUT size of {}MB. the maximum will be used instead",
                    config.getMpuThresholdMb(), MAX_PUT_SIZE_MB);
            config.setMpuThresholdMb(MAX_PUT_SIZE_MB);
        }
        if (config.getMpuPartSizeMb() < MIN_PART_SIZE_MB) {
            log.warn("{}MB is below the minimum MPU part size of {}MB. the minimum will be used instead",
                    config.getMpuPartSizeMb(), MIN_PART_SIZE_MB);
            config.setMpuPartSizeMb(MIN_PART_SIZE_MB);
        }

        if (source != null) sourceReadWindow = source.getReadWindow();
    }

    // make sure bucket exists
    if (!bucketExists)
        throw new ConfigurationException("The bucket " + config.getBucketName() + " does not exist.");

    // if syncing versions, make sure plugins support it and bucket has versioning enabled
    if (config.isIncludeVersions()) {
        if (!(source instanceof AbstractS3Storage && target instanceof AbstractS3Storage))
            throw new ConfigurationException("Version migration is only supported between two S3 plugins");

        if (!bucketHasVersions)
            throw new ConfigurationException("The specified bucket does not have versioning enabled.");
    }
}
 
Example 15
Source File: AbstractAWSProcessor.java    From nifi with Apache License 2.0 4 votes vote down vote up
protected ClientConfiguration createConfiguration(final ProcessContext context) {
    final ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(context.getMaxConcurrentTasks());
    config.setMaxErrorRetry(0);
    config.setUserAgent(DEFAULT_USER_AGENT);
    // If this is changed to be a property, ensure other uses are also changed
    config.setProtocol(DEFAULT_PROTOCOL);
    final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    config.setConnectionTimeout(commsTimeout);
    config.setSocketTimeout(commsTimeout);

    if(this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) {
        final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
        if (sslContextService != null) {
            final SSLContext sslContext = sslContextService.createSSLContext(SslContextFactory.ClientAuth.NONE);
            // NIFI-3788: Changed hostnameVerifier from null to DHV (BrowserCompatibleHostnameVerifier is deprecated)
            SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, new DefaultHostnameVerifier());
            config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory);
        }
    }

    final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context, () -> {
        if (context.getProperty(PROXY_HOST).isSet()) {
            final ProxyConfiguration componentProxyConfig = new ProxyConfiguration();
            String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue();
            Integer proxyPort = context.getProperty(PROXY_HOST_PORT).evaluateAttributeExpressions().asInteger();
            String proxyUsername = context.getProperty(PROXY_USERNAME).evaluateAttributeExpressions().getValue();
            String proxyPassword = context.getProperty(PROXY_PASSWORD).evaluateAttributeExpressions().getValue();
            componentProxyConfig.setProxyType(Proxy.Type.HTTP);
            componentProxyConfig.setProxyServerHost(proxyHost);
            componentProxyConfig.setProxyServerPort(proxyPort);
            componentProxyConfig.setProxyUserName(proxyUsername);
            componentProxyConfig.setProxyUserPassword(proxyPassword);
            return componentProxyConfig;
        }
        return ProxyConfiguration.DIRECT_CONFIGURATION;
    });

    if (Proxy.Type.HTTP.equals(proxyConfig.getProxyType())) {
        config.setProxyHost(proxyConfig.getProxyServerHost());
        config.setProxyPort(proxyConfig.getProxyServerPort());

        if (proxyConfig.hasCredential()) {
            config.setProxyUsername(proxyConfig.getProxyUserName());
            config.setProxyPassword(proxyConfig.getProxyUserPassword());
        }
    }

    return config;
}
 
Example 16
Source File: S3Utils.java    From cloudstack with Apache License 2.0 4 votes vote down vote up
public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }
 
Example 17
Source File: AWSDatabaseHolder.java    From billow with Apache License 2.0 4 votes vote down vote up
public AWSDatabaseHolder(Config config) {
    maxAgeInMs = config.getDuration("maxAge", TimeUnit.MILLISECONDS);

    final DefaultAWSCredentialsProviderChain awsCredentialsProviderChain = new DefaultAWSCredentialsProviderChain();

    final ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setRetryPolicy(new RetryPolicy(null, null, config.getInt("maxErrorRetry"), true));
    clientConfig.setSocketTimeout(config.getInt("socketTimeout") * 1000);

    final AmazonEC2 bootstrapEC2Client = AmazonEC2ClientBuilder.standard().withCredentials(awsCredentialsProviderChain).build();

    ec2Clients = Maps.newHashMap();
    rdsClients = Maps.newHashMap();
    sqsClients = Maps.newHashMap();
    dynamoDBClients = Maps.newHashMap();
    elasticacheClients = Maps.newHashMap();
    elasticsearchClients = Maps.newHashMap();

    final List<Region> ec2Regions = bootstrapEC2Client.describeRegions().getRegions();
    for (Region region : ec2Regions) {
        final String regionName = region.getRegionName();
        final String endpoint = region.getEndpoint();
        log.debug("Adding ec2 region {}", region);

        if (config.getBoolean("ec2Enabled")) {
            final AmazonEC2Client ec2Client = new AmazonEC2Client(awsCredentialsProviderChain, clientConfig);
            ec2Client.setEndpoint(endpoint);
            ec2Clients.put(regionName, ec2Client);
        }

        if (config.getBoolean("rdsEnabled")) {
            final AmazonRDSClient rdsClient = new AmazonRDSClient(awsCredentialsProviderChain, clientConfig);
            rdsClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "rds."));
            rdsClients.put(regionName, rdsClient);
        }

        if (config.getBoolean("dynamodbEnabled")) {
            final AmazonDynamoDBClient dynamoDBClient =
                new AmazonDynamoDBClient(awsCredentialsProviderChain, clientConfig);
            dynamoDBClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "dynamodb."));
            dynamoDBClients.put(regionName, dynamoDBClient);
        }

        if (config.getBoolean("sqsEnabled")) {
            final AmazonSQSClient sqsClient = new AmazonSQSClient(awsCredentialsProviderChain, clientConfig);
            sqsClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "sqs."));
            sqsClients.put(regionName, sqsClient);
        }

        if (config.getBoolean("elasticacheEnabled")) {
            final AmazonElastiCacheClient elastiCacheClient = new AmazonElastiCacheClient
                (awsCredentialsProviderChain, clientConfig);
            elastiCacheClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "elasticache."));
            elasticacheClients.put(regionName, elastiCacheClient);
        }

        if (config.getBoolean("elasticsearchEnabled")) {
            final AWSElasticsearchClient elasticsearchClient = new AWSElasticsearchClient
                (awsCredentialsProviderChain, clientConfig);
            elasticsearchClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "es."));
            elasticsearchClients.put(regionName, elasticsearchClient);
        }
    }

    this.iamClient = AmazonIdentityManagementClientBuilder.standard()
        .withCredentials(awsCredentialsProviderChain)
        .withClientConfiguration(clientConfig)
        .build();

    if (config.hasPath("accountNumber")) {
        this.awsAccountNumber = config.getString("accountNumber");
    } else {
        this.awsAccountNumber = null;
    }

    if (config.hasPath("arnPartition")) {
        this.awsARNPartition = config.getString("arnPartition");
    } else {
        this.awsARNPartition = "aws";
    }

    rebuild();
}