Java Code Examples for org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: KinesisConfigUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * <p>
 *  A set of configuration paremeters associated with the describeStreams API may be used if:
 * 	1) an legacy client wants to consume from Kinesis
 * 	2) a current client wants to consumer from DynamoDB streams
 *
 * In the context of 1), the set of configurations needs to be translated to the corresponding
 * configurations in the Kinesis listShards API. In the mean time, keep these configs since
 * they are applicable in the context of 2), i.e., polling data from a DynamoDB stream.
 * </p>
 *
 * @param configProps original config properties.
 * @return backfilled config properties.
 */
public static Properties backfillConsumerKeys(Properties configProps) {
	HashMap<String, String> oldKeyToNewKeys = new HashMap<>();
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_BASE, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_MAX, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_EXPONENTIAL_CONSTANT, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_EXPONENTIAL_CONSTANT);
	for (Map.Entry<String, String> entry : oldKeyToNewKeys.entrySet()) {
		String oldKey = entry.getKey();
		String newKey = entry.getValue();
		if (configProps.containsKey(oldKey)) {
			configProps.setProperty(newKey, configProps.getProperty(oldKey));
			// Do not remove the oldKey since they may be used in the context of talking to DynamoDB streams
		}
	}
	return configProps;
}
 
Example 3
Source Project: Flink-CEPplus   Source File: ConsumeFromKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 
Example 4
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 6
Source Project: flink   Source File: ConsumeFromKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 
Example 7
Source Project: flink   Source File: ShardConsumer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Calculates how many records to read each time through the loop based on a target throughput
 * and the measured frequenecy of the loop.
 * @param runLoopTimeNanos The total time of one pass through the loop
 * @param numRecords The number of records of the last read operation
 * @param recordBatchSizeBytes The total batch size of the last read operation
 * @param maxNumberOfRecordsPerFetch The current maxNumberOfRecordsPerFetch
 */
private int adaptRecordsToRead(long runLoopTimeNanos, int numRecords, long recordBatchSizeBytes,
		int maxNumberOfRecordsPerFetch) {
	if (useAdaptiveReads && numRecords != 0 && runLoopTimeNanos != 0) {
		long averageRecordSizeBytes = recordBatchSizeBytes / numRecords;
		// Adjust number of records to fetch from the shard depending on current average record size
		// to optimize 2 Mb / sec read limits
		double loopFrequencyHz = 1000000000.0d / runLoopTimeNanos;
		double bytesPerRead = KINESIS_SHARD_BYTES_PER_SECOND_LIMIT / loopFrequencyHz;
		maxNumberOfRecordsPerFetch = (int) (bytesPerRead / averageRecordSizeBytes);
		// Ensure the value is greater than 0 and not more than 10000L
		maxNumberOfRecordsPerFetch = Math.max(1, Math.min(maxNumberOfRecordsPerFetch, ConsumerConfigConstants.DEFAULT_SHARD_GETRECORDS_MAX));

		// Set metrics
		shardMetricsReporter.setAverageRecordSizeBytes(averageRecordSizeBytes);
		shardMetricsReporter.setLoopFrequencyHz(loopFrequencyHz);
		shardMetricsReporter.setBytesPerRead(bytesPerRead);
	}
	return maxNumberOfRecordsPerFetch;
}
 
Example 8
Source Project: flink   Source File: ConsumeFromDynamoDBStreams.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties dynamodbStreamsConsumerConfig = new Properties();
	final String streamName = pt.getRequired(DYNAMODB_STREAM_NAME);
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> dynamodbStreams = see.addSource(new FlinkDynamoDBStreamsConsumer<>(
			streamName,
			new SimpleStringSchema(),
			dynamodbStreamsConsumerConfig));

	dynamodbStreams.print();

	see.execute();
}
 
Example 9
Source Project: flink   Source File: KinesisConfigUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * <p>
 *  A set of configuration paremeters associated with the describeStreams API may be used if:
 * 	1) an legacy client wants to consume from Kinesis
 * 	2) a current client wants to consumer from DynamoDB streams
 *
 * In the context of 1), the set of configurations needs to be translated to the corresponding
 * configurations in the Kinesis listShards API. In the mean time, keep these configs since
 * they are applicable in the context of 2), i.e., polling data from a DynamoDB stream.
 * </p>
 *
 * @param configProps original config properties.
 * @return backfilled config properties.
 */
public static Properties backfillConsumerKeys(Properties configProps) {
	HashMap<String, String> oldKeyToNewKeys = new HashMap<>();
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_BASE, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_MAX, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_EXPONENTIAL_CONSTANT, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_EXPONENTIAL_CONSTANT);
	for (Map.Entry<String, String> entry : oldKeyToNewKeys.entrySet()) {
		String oldKey = entry.getKey();
		String newKey = entry.getValue();
		if (configProps.containsKey(oldKey)) {
			configProps.setProperty(newKey, configProps.getProperty(oldKey));
			// Do not remove the oldKey since they may be used in the context of talking to DynamoDB streams
		}
	}
	return configProps;
}
 
Example 10
Source Project: flink   Source File: ConsumeFromKinesis.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 
Example 11
Source Project: flink   Source File: ConsumeFromDynamoDBStreams.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties dynamodbStreamsConsumerConfig = new Properties();
	final String streamName = pt.getRequired(DYNAMODB_STREAM_NAME);
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> dynamodbStreams = see.addSource(new FlinkDynamoDBStreamsConsumer<>(
			streamName,
			new SimpleStringSchema(),
			dynamodbStreamsConsumerConfig));

	dynamodbStreams.print();

	see.execute();
}
 
Example 12
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 13
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 14
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 
Example 15
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 6 votes vote down vote up
public SingleShardEmittingAdaptiveNumOfRecordsKinesis(final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	this.totalNumOfRecords = numOfRecords;
	this.totalNumOfGetRecordsCalls = numOfGetRecordsCalls;
	this.millisBehindLatest = millisBehindLatest;
	this.averageRecordSizeBytes = 0L;

	// initialize the record batches that we will be fetched
	this.shardItrToRecordBatch = new HashMap<>();

	int numOfAlreadyPartitionedRecords = 0;
	int numOfRecordsPerBatch = numOfRecords;
	for (int batch = 0; batch < totalNumOfGetRecordsCalls; batch++) {
			shardItrToRecordBatch.put(
					String.valueOf(batch),
					createRecordBatchWithRange(
							numOfAlreadyPartitionedRecords,
							numOfAlreadyPartitionedRecords + numOfRecordsPerBatch));
			numOfAlreadyPartitionedRecords += numOfRecordsPerBatch;

		numOfRecordsPerBatch = (int) (KINESIS_SHARD_BYTES_PER_SECOND_LIMIT /
				(averageRecordSizeBytes * 1000L / ConsumerConfigConstants.DEFAULT_SHARD_GETRECORDS_INTERVAL_MILLIS));
	}
}
 
Example 16
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForListShardsBackoffMaxMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for list shards operation max backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 17
Source Project: Flink-CEPplus   Source File: KinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
private RecordEmitter createRecordEmitter(Properties configProps) {
	if (periodicWatermarkAssigner != null && watermarkTracker != null) {
		int queueCapacity = Integer.parseInt(configProps.getProperty(
			ConsumerConfigConstants.WATERMARK_SYNC_QUEUE_CAPACITY,
			Integer.toString(AsyncKinesisRecordEmitter.DEFAULT_QUEUE_CAPACITY)));
		return new AsyncKinesisRecordEmitter(queueCapacity);
	}
	return new SyncKinesisRecordEmitter();
}
 
Example 18
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidPatternForInitialTimestampInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for initial timestamp for AT_TIMESTAMP initial position in stream.");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, "2016-03-14");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_TIMESTAMP_DATE_FORMAT, "InvalidPattern");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 19
Source Project: Flink-CEPplus   Source File: KinesisProxyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 
Example 20
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForGetRecordsBackoffBaseMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for get records operation base backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_BACKOFF_BASE, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 21
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStreamInitPositionTypeSetToAtTimestampButNoInitTimestampSetInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Please set value for initial timestamp ('"
			+ ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP + "') when using AT_TIMESTAMP initial position.");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableDateForInitialTimestampInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for initial timestamp for AT_TIMESTAMP initial position in stream.");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, "unparsableDate");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 23
Source Project: flink   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableIntForGetRecordsMaxCountInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum records per getRecords shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_MAX, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 24
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableDateForUserDefinedDateFormatForInitialTimestampInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for initial timestamp for AT_TIMESTAMP initial position in stream.");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, "stillUnparsable");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_TIMESTAMP_DATE_FORMAT, "yyyy-MM-dd");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 25
Source Project: flink   Source File: KinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
private RecordEmitter createRecordEmitter(Properties configProps) {
	if (periodicWatermarkAssigner != null && watermarkTracker != null) {
		int queueCapacity = Integer.parseInt(configProps.getProperty(
			ConsumerConfigConstants.WATERMARK_SYNC_QUEUE_CAPACITY,
			Integer.toString(AsyncKinesisRecordEmitter.DEFAULT_QUEUE_CAPACITY)));
		return new AsyncKinesisRecordEmitter(queueCapacity);
	}
	return new SyncKinesisRecordEmitter();
}
 
Example 26
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForListShardsBackoffBaseMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for list shards operation base backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 27
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableLongForListShardsBackoffMaxMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for list shards operation max backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 28
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableDoubleForListShardsBackoffExponentialConstantInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for list shards operation backoff exponential constant");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_BACKOFF_EXPONENTIAL_CONSTANT, "unparsableDouble");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 29
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableIntForGetRecordsRetriesInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum retry attempts for getRecords shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_RETRIES, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 
Example 30
Source Project: Flink-CEPplus   Source File: KinesisConfigUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnparsableIntForGetRecordsMaxCountInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum records per getRecords shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_MAX, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}