Java Code Examples for org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: AWSUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: ProduceIntoKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}
 
Example 3
Source Project: flink   Source File: AWSUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example 4
Source Project: flink   Source File: ProduceIntoKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}
 
Example 5
Source Project: flink-statefun   Source File: AwsAuthConfigProperties.java    License: Apache License 2.0 6 votes vote down vote up
static Properties forAwsRegionConsumerProps(AwsRegion awsRegion) {
  final Properties properties = new Properties();

  if (awsRegion.isDefault()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, regionFromDefaultProviderChain());
  } else if (awsRegion.isId()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, awsRegion.asId().id());
  } else if (awsRegion.isCustomEndpoint()) {
    final AwsRegion.CustomEndpointAwsRegion customEndpoint = awsRegion.asCustomEndpoint();
    properties.setProperty(AWSConfigConstants.AWS_ENDPOINT, customEndpoint.serviceEndpoint());
    properties.setProperty(AWSConfigConstants.AWS_REGION, customEndpoint.regionId());
  } else {
    throw new IllegalStateException("Unrecognized AWS region configuration type: " + awsRegion);
  }

  return properties;
}
 
Example 6
Source Project: flink-statefun   Source File: AwsAuthConfigProperties.java    License: Apache License 2.0 6 votes vote down vote up
static Properties forAwsRegionProducerProps(AwsRegion awsRegion) {
  final Properties properties = new Properties();

  if (awsRegion.isDefault()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, regionFromDefaultProviderChain());
  } else if (awsRegion.isId()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, awsRegion.asId().id());
  } else if (awsRegion.isCustomEndpoint()) {
    final AwsRegion.CustomEndpointAwsRegion customEndpoint = awsRegion.asCustomEndpoint();

    final URI uri = URI.create(customEndpoint.serviceEndpoint());
    properties.setProperty("KinesisEndpoint", uri.getHost());
    properties.setProperty(AWSConfigConstants.AWS_REGION, customEndpoint.regionId());

    int port = uri.getPort();
    if (port != -1) {
      properties.setProperty("KinesisPort", String.valueOf(port));
    }
  } else {
    throw new IllegalStateException("Unrecognized AWS region configuration type: " + awsRegion);
  }

  return properties;
}
 
Example 7
@Test
public void awsBasicCredentialsProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsCredentials(
          AwsCredentials.basic("fake-access-key-id", "fake-secret-access-key"));

  assertThat(properties.entrySet(), hasSize(3));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.AWS_CREDENTIALS_PROVIDER,
          AWSConfigConstants.CredentialProvider.BASIC.name()));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.accessKeyId(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-access-key-id"));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.secretKey(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-secret-access-key"));
}
 
Example 8
@Test
public void awsProfileCredentialsProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsCredentials(
          AwsCredentials.profile("fake-profile", "/fake/profile/path"));

  assertThat(properties.entrySet(), hasSize(3));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.AWS_CREDENTIALS_PROVIDER,
          AWSConfigConstants.CredentialProvider.PROFILE.name()));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.profileName(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-profile"));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.profilePath(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "/fake/profile/path"));
}
 
Example 9
Source Project: flink   Source File: AWSUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// If an endpoint is specified, we give preference to using an endpoint and use the region property to
		// sign the request.
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
			configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
			configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example 10
Source Project: flink   Source File: ProduceIntoKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}
 
Example 11
Source Project: Flink-CEPplus   Source File: KinesisConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 
Example 12
Source Project: flink   Source File: TestUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get standard Kinesis-related config properties.
 */
public static Properties getStandardProperties() {
	Properties config = new Properties();
	config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
	config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	return config;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: KinesisProxyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testClientConfigOverride() {

	Properties configProps = new Properties();
	configProps.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	configProps.setProperty(AWSUtil.AWS_CLIENT_CONFIG_PREFIX + "socketTimeout", "9999");

	KinesisProxyInterface proxy = KinesisProxy.create(configProps);

	AmazonKinesis kinesisClient = Whitebox.getInternalState(proxy, "kinesisClient");
	ClientConfiguration clientConfiguration = Whitebox.getInternalState(kinesisClient,
		"clientConfiguration");
	assertEquals(9999, clientConfiguration.getSocketTimeout());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForProducerConfiguration() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Error trying to set field RateLimit with the value 'unparsableLong'");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty("RateLimit", "unparsableLong");

	KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 
Example 16
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 
Example 17
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 
Example 18
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReplaceDeprecatedKeys() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	// these deprecated keys should be replaced
	testConfig.setProperty(ProducerConfigConstants.AGGREGATION_MAX_COUNT, "1");
	testConfig.setProperty(ProducerConfigConstants.COLLECTION_MAX_COUNT, "2");
	Properties replacedConfig = KinesisConfigUtil.replaceDeprecatedProducerKeys(testConfig);

	assertEquals("1", replacedConfig.getProperty(KinesisConfigUtil.AGGREGATION_MAX_COUNT));
	assertEquals("2", replacedConfig.getProperty(KinesisConfigUtil.COLLECTION_MAX_COUNT));
}
 
Example 19
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 
Example 20
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMissingAwsRegionInProducerConfig() {
	String expectedMessage = String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.",
			AWSConfigConstants.AWS_REGION);
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage(expectedMessage);

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
}
 
Example 21
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnrecognizableAwsRegionInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid AWS region");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "wrongRegionId");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCredentialProviderTypeSetToBasicButNoCredentialSetInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Please set values for AWS Access Key ID ('" + AWSConfigConstants.AWS_ACCESS_KEY_ID + "') " +
			"and Secret Key ('" + AWSConfigConstants.AWS_SECRET_ACCESS_KEY + "') when using the BASIC AWS credential provider type.");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnrecognizableCredentialProviderTypeInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid AWS Credential Provider Type");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "wrongProviderType");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}
 
Example 24
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAwsRegionOrEndpointInConsumerConfig() {
	String expectedMessage = String.format("For FlinkKinesisConsumer either AWS region ('%s') or AWS endpoint ('%s') must be set in the config.",
			AWSConfigConstants.AWS_REGION, AWSConfigConstants.AWS_ENDPOINT);
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage(expectedMessage);

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_ENDPOINT, "fake");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 25
Source Project: Flink-CEPplus   Source File: TestUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get standard Kinesis-related config properties.
 */
public static Properties getStandardProperties() {
	Properties config = new Properties();
	config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
	config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	return config;
}
 
Example 26
Source Project: flink   Source File: KinesisConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 
Example 27
Source Project: flink   Source File: KinesisProxyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testClientConfigOverride() {

	Properties configProps = new Properties();
	configProps.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	configProps.setProperty(AWSUtil.AWS_CLIENT_CONFIG_PREFIX + "socketTimeout", "9999");

	KinesisProxyInterface proxy = KinesisProxy.create(configProps);

	AmazonKinesis kinesisClient = Whitebox.getInternalState(proxy, "kinesisClient");
	ClientConfiguration clientConfiguration = Whitebox.getInternalState(kinesisClient,
		"clientConfiguration");
	assertEquals(9999, clientConfiguration.getSocketTimeout());
}
 
Example 28
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForProducerConfiguration() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Error trying to set field RateLimit with the value 'unparsableLong'");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty("RateLimit", "unparsableLong");

	KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);
}
 
Example 29
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 
Example 30
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}