org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.config.AWSConfigConstants. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

6 votes

@Test
public void awsProfileCredentialsProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsCredentials(
          AwsCredentials.profile("fake-profile", "/fake/profile/path"));

  assertThat(properties.entrySet(), hasSize(3));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.AWS_CREDENTIALS_PROVIDER,
          AWSConfigConstants.CredentialProvider.PROFILE.name()));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.profileName(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-profile"));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.profilePath(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "/fake/profile/path"));
}

Example #2

Source File: AWSUtil.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}

Example #3

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

6 votes

@Test
public void awsBasicCredentialsProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsCredentials(
          AwsCredentials.basic("fake-access-key-id", "fake-secret-access-key"));

  assertThat(properties.entrySet(), hasSize(3));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.AWS_CREDENTIALS_PROVIDER,
          AWSConfigConstants.CredentialProvider.BASIC.name()));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.accessKeyId(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-access-key-id"));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.secretKey(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER),
          "fake-secret-access-key"));
}

Example #4

Source File: AWSUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}

Example #5

Source File: ProduceIntoKinesis.java From Flink-CEPplus with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}

Example #6

Source File: AwsAuthConfigProperties.java From flink-statefun with Apache License 2.0

6 votes

static Properties forAwsRegionConsumerProps(AwsRegion awsRegion) {
  final Properties properties = new Properties();

  if (awsRegion.isDefault()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, regionFromDefaultProviderChain());
  } else if (awsRegion.isId()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, awsRegion.asId().id());
  } else if (awsRegion.isCustomEndpoint()) {
    final AwsRegion.CustomEndpointAwsRegion customEndpoint = awsRegion.asCustomEndpoint();
    properties.setProperty(AWSConfigConstants.AWS_ENDPOINT, customEndpoint.serviceEndpoint());
    properties.setProperty(AWSConfigConstants.AWS_REGION, customEndpoint.regionId());
  } else {
    throw new IllegalStateException("Unrecognized AWS region configuration type: " + awsRegion);
  }

  return properties;
}

Example #7

Source File: AwsAuthConfigProperties.java From flink-statefun with Apache License 2.0

6 votes

static Properties forAwsRegionProducerProps(AwsRegion awsRegion) {
  final Properties properties = new Properties();

  if (awsRegion.isDefault()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, regionFromDefaultProviderChain());
  } else if (awsRegion.isId()) {
    properties.setProperty(AWSConfigConstants.AWS_REGION, awsRegion.asId().id());
  } else if (awsRegion.isCustomEndpoint()) {
    final AwsRegion.CustomEndpointAwsRegion customEndpoint = awsRegion.asCustomEndpoint();

    final URI uri = URI.create(customEndpoint.serviceEndpoint());
    properties.setProperty("KinesisEndpoint", uri.getHost());
    properties.setProperty(AWSConfigConstants.AWS_REGION, customEndpoint.regionId());

    int port = uri.getPort();
    if (port != -1) {
      properties.setProperty("KinesisPort", String.valueOf(port));
    }
  } else {
    throw new IllegalStateException("Unrecognized AWS region configuration type: " + awsRegion);
  }

  return properties;
}

Example #8

Source File: ProduceIntoKinesis.java From flink with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}

Example #9

Source File: AWSUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// If an endpoint is specified, we give preference to using an endpoint and use the region property to
		// sign the request.
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
			configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
			configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}

Example #10

Source File: ProduceIntoKinesis.java From flink with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	DataStream<String> simpleStringStream = see.addSource(new EventsGenerator());

	Properties kinesisProducerConfig = new Properties();
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accessKey"));
	kinesisProducerConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretKey"));

	FlinkKinesisProducer<String> kinesis = new FlinkKinesisProducer<>(
			new SimpleStringSchema(), kinesisProducerConfig);

	kinesis.setFailOnError(true);
	kinesis.setDefaultStream("flink-test");
	kinesis.setDefaultPartition("0");

	simpleStringStream.addSink(kinesis);

	see.execute();
}

Example #11

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNoAwsRegionOrEndpointInConsumerConfig() {
	String expectedMessage = String.format("For FlinkKinesisConsumer AWS region ('%s') and/or AWS endpoint ('%s') must be set in the config.",
			AWSConfigConstants.AWS_REGION, AWSConfigConstants.AWS_ENDPOINT);
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage(expectedMessage);

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}

Example #12

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAwsRegionInConsumerConfig() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}

Example #13

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testCredentialProviderTypeSetToBasicButNoCredentialSetInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Please set values for AWS Access Key ID ('" + AWSConfigConstants.AWS_ACCESS_KEY_ID + "') " +
			"and Secret Key ('" + AWSConfigConstants.AWS_SECRET_ACCESS_KEY + "') when using the BASIC AWS credential provider type.");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}

Example #14

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testUnrecognizableCredentialProviderTypeInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid AWS Credential Provider Type");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "wrongProviderType");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}

Example #15

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsDefaultCredentialsProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsCredentials(AwsCredentials.fromDefaultProviderChain());

  assertThat(properties.entrySet(), hasSize(1));
  assertThat(
      properties,
      hasEntry(
          AWSConfigConstants.AWS_CREDENTIALS_PROVIDER,
          AWSConfigConstants.CredentialProvider.AUTO.name()));
}

Example #16

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAwsRegionAndEndpointInConsumerConfig() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_ENDPOINT, "fake");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}

Example #17

Source File: AWSUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testGetCredentialsProvider() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "WEB_IDENTITY_TOKEN");

	AWSCredentialsProvider credentialsProvider = AWSUtil.getCredentialsProvider(testConfig);
	assertTrue(credentialsProvider instanceof WebIdentityTokenCredentialsProvider);
}

Example #18

Source File: AWSUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testInvalidCredentialsProvider() {
	exception.expect(IllegalArgumentException.class);

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "INVALID_PROVIDER");

	AWSUtil.getCredentialsProvider(testConfig);
}

Example #19

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testUnrecognizableAwsRegionInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid AWS region");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "wrongRegionId");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}

Example #20

Source File: KinesisConfigUtil.java From flink with Apache License 2.0

5 votes

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}

Example #21

Source File: KinesisProxyTest.java From flink with Apache License 2.0

5 votes

@Test
public void testClientConfigOverride() {

	Properties configProps = new Properties();
	configProps.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	configProps.setProperty(AWSUtil.AWS_CLIENT_CONFIG_PREFIX + "socketTimeout", "9999");

	KinesisProxyInterface proxy = KinesisProxy.create(configProps);

	AmazonKinesis kinesisClient = Whitebox.getInternalState(proxy, "kinesisClient");
	ClientConfiguration clientConfiguration = Whitebox.getInternalState(kinesisClient,
		"clientConfiguration");
	assertEquals(9999, clientConfiguration.getSocketTimeout());
}

Example #22

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsCustomEndpointRegionProducerProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsRegionProducerProps(
          AwsRegion.ofCustomEndpoint("https://foo.bar:6666", "us-east-1"));

  assertThat(properties.entrySet(), hasSize(3));
  assertThat(properties, hasEntry("KinesisEndpoint", "foo.bar"));
  assertThat(properties, hasEntry("KinesisPort", "6666"));
  assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-east-1"));
}

Example #23

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsSpecificRegionProducerProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsRegionProducerProps(AwsRegion.ofId("us-east-2"));

  assertThat(properties.entrySet(), hasSize(1));
  assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-east-2"));
}

Example #24

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsDefaultRegionProducerProperties() {
  // TODO Flink doesn't support auto region detection from the AWS provider chain,
  // TODO so we always have to have the region settings available in the client side
  // TODO this should no longer be a restriction once we fix this in the Flink connector side
  try (final ScopedSystemProperty awsRegionSystemProps =
      new ScopedSystemProperty(SDKGlobalConfiguration.AWS_REGION_SYSTEM_PROPERTY, "us-west-1")) {
    final Properties properties =
        AwsAuthConfigProperties.forAwsRegionProducerProps(AwsRegion.fromDefaultProviderChain());

    assertThat(properties.entrySet(), hasSize(1));
    assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-west-1"));
  }
}

Example #25

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsCustomEndpointRegionConsumerProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsRegionConsumerProps(
          AwsRegion.ofCustomEndpoint("https://foo.bar:6666", "us-east-1"));

  assertThat(properties.entrySet(), hasSize(2));
  assertThat(properties, hasEntry(AWSConfigConstants.AWS_ENDPOINT, "https://foo.bar:6666"));
  assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-east-1"));
}

Example #26

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsSpecificRegionConsumerProperties() {
  final Properties properties =
      AwsAuthConfigProperties.forAwsRegionConsumerProps(AwsRegion.ofId("us-east-2"));

  assertThat(properties.entrySet(), hasSize(1));
  assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-east-2"));
}

Example #27

Source File: AwsAuthConfigPropertiesTest.java From flink-statefun with Apache License 2.0

5 votes

@Test
public void awsDefaultRegionConsumerProperties() {
  // TODO Flink doesn't support auto region detection from the AWS provider chain,
  // TODO so we always have to have the region settings available in the client side
  // TODO this should no longer be a restriction once we fix this in the Flink connector side
  try (final ScopedSystemProperty awsRegionSystemProps =
      new ScopedSystemProperty(SDKGlobalConfiguration.AWS_REGION_SYSTEM_PROPERTY, "us-west-1")) {
    final Properties properties =
        AwsAuthConfigProperties.forAwsRegionConsumerProps(AwsRegion.fromDefaultProviderChain());

    assertThat(properties.entrySet(), hasSize(1));
    assertThat(properties, hasEntry(AWSConfigConstants.AWS_REGION, "us-west-1"));
  }
}

Example #28

Source File: TestUtils.java From flink with Apache License 2.0

5 votes

/**
 * Get standard Kinesis-related config properties.
 */
public static Properties getStandardProperties() {
	Properties config = new Properties();
	config.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	config.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
	config.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	return config;
}

Example #29

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAwsRegionOrEndpointInConsumerConfig() {
	String expectedMessage = String.format("For FlinkKinesisConsumer either AWS region ('%s') or AWS endpoint ('%s') must be set in the config.",
			AWSConfigConstants.AWS_REGION, AWSConfigConstants.AWS_ENDPOINT);
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage(expectedMessage);

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_ENDPOINT, "fake");
	testConfig.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, "accessKey");
	testConfig.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}

Example #30

Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testCredentialProviderTypeSetToBasicButNoCredentialSetInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Please set values for AWS Access Key ID ('" + AWSConfigConstants.AWS_ACCESS_KEY_ID + "') " +
			"and Secret Key ('" + AWSConfigConstants.AWS_SECRET_ACCESS_KEY + "') when using the BASIC AWS credential provider type.");

	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	testConfig.setProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");

	KinesisConfigUtil.validateAwsConfiguration(testConfig);
}