org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants Java Examples

/**
 * Calculates how many records to read each time through the loop based on a target throughput
 * and the measured frequenecy of the loop.
 * @param runLoopTimeNanos The total time of one pass through the loop
 * @param numRecords The number of records of the last read operation
 * @param recordBatchSizeBytes The total batch size of the last read operation
 * @param maxNumberOfRecordsPerFetch The current maxNumberOfRecordsPerFetch
 */
private int adaptRecordsToRead(long runLoopTimeNanos, int numRecords, long recordBatchSizeBytes,
		int maxNumberOfRecordsPerFetch) {
	if (useAdaptiveReads && numRecords != 0 && runLoopTimeNanos != 0) {
		long averageRecordSizeBytes = recordBatchSizeBytes / numRecords;
		// Adjust number of records to fetch from the shard depending on current average record size
		// to optimize 2 Mb / sec read limits
		double loopFrequencyHz = 1000000000.0d / runLoopTimeNanos;
		double bytesPerRead = KINESIS_SHARD_BYTES_PER_SECOND_LIMIT / loopFrequencyHz;
		maxNumberOfRecordsPerFetch = (int) (bytesPerRead / averageRecordSizeBytes);
		// Ensure the value is greater than 0 and not more than 10000L
		maxNumberOfRecordsPerFetch = Math.max(1, Math.min(maxNumberOfRecordsPerFetch, ConsumerConfigConstants.DEFAULT_SHARD_GETRECORDS_MAX));

		// Set metrics
		shardMetricsReporter.setAverageRecordSizeBytes(averageRecordSizeBytes);
		shardMetricsReporter.setLoopFrequencyHz(loopFrequencyHz);
		shardMetricsReporter.setBytesPerRead(bytesPerRead);
	}
	return maxNumberOfRecordsPerFetch;
}
 

@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 

@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

@Test
public void testDateStringForValidateOptionDateProperty() {
	String timestamp = "2016-04-04T19:58:46.480-00:00";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, timestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties dynamodbStreamsConsumerConfig = new Properties();
	final String streamName = pt.getRequired(DYNAMODB_STREAM_NAME);
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> dynamodbStreams = see.addSource(new FlinkDynamoDBStreamsConsumer<>(
			streamName,
			new SimpleStringSchema(),
			dynamodbStreamsConsumerConfig));

	dynamodbStreams.print();

	see.execute();
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties dynamodbStreamsConsumerConfig = new Properties();
	final String streamName = pt.getRequired(DYNAMODB_STREAM_NAME);
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	dynamodbStreamsConsumerConfig.setProperty(
			ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> dynamodbStreams = see.addSource(new FlinkDynamoDBStreamsConsumer<>(
			streamName,
			new SimpleStringSchema(),
			dynamodbStreamsConsumerConfig));

	dynamodbStreams.print();

	see.execute();
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 

public SingleShardEmittingAdaptiveNumOfRecordsKinesis(final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	this.totalNumOfRecords = numOfRecords;
	this.totalNumOfGetRecordsCalls = numOfGetRecordsCalls;
	this.millisBehindLatest = millisBehindLatest;
	this.averageRecordSizeBytes = 0L;

	// initialize the record batches that we will be fetched
	this.shardItrToRecordBatch = new HashMap<>();

	int numOfAlreadyPartitionedRecords = 0;
	int numOfRecordsPerBatch = numOfRecords;
	for (int batch = 0; batch < totalNumOfGetRecordsCalls; batch++) {
			shardItrToRecordBatch.put(
					String.valueOf(batch),
					createRecordBatchWithRange(
							numOfAlreadyPartitionedRecords,
							numOfAlreadyPartitionedRecords + numOfRecordsPerBatch));
			numOfAlreadyPartitionedRecords += numOfRecordsPerBatch;

		numOfRecordsPerBatch = (int) (KINESIS_SHARD_BYTES_PER_SECOND_LIMIT /
				(averageRecordSizeBytes * 1000L / ConsumerConfigConstants.DEFAULT_SHARD_GETRECORDS_INTERVAL_MILLIS));
	}
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(1);

	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, pt.getRequired("region"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, pt.getRequired("accesskey"));
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, pt.getRequired("secretkey"));

	DataStream<String> kinesis = see.addSource(new FlinkKinesisConsumer<>(
		"flink-test",
		new SimpleStringSchema(),
		kinesisConsumerConfig));

	kinesis.print();

	see.execute();
}
 

/**
 * <p>
 *  A set of configuration paremeters associated with the describeStreams API may be used if:
 * 	1) an legacy client wants to consume from Kinesis
 * 	2) a current client wants to consumer from DynamoDB streams
 *
 * In the context of 1), the set of configurations needs to be translated to the corresponding
 * configurations in the Kinesis listShards API. In the mean time, keep these configs since
 * they are applicable in the context of 2), i.e., polling data from a DynamoDB stream.
 * </p>
 *
 * @param configProps original config properties.
 * @return backfilled config properties.
 */
public static Properties backfillConsumerKeys(Properties configProps) {
	HashMap<String, String> oldKeyToNewKeys = new HashMap<>();
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_BASE, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_MAX, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_EXPONENTIAL_CONSTANT, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_EXPONENTIAL_CONSTANT);
	for (Map.Entry<String, String> entry : oldKeyToNewKeys.entrySet()) {
		String oldKey = entry.getKey();
		String newKey = entry.getValue();
		if (configProps.containsKey(oldKey)) {
			configProps.setProperty(newKey, configProps.getProperty(oldKey));
			// Do not remove the oldKey since they may be used in the context of talking to DynamoDB streams
		}
	}
	return configProps;
}
 

/**
 * <p>
 *  A set of configuration paremeters associated with the describeStreams API may be used if:
 * 	1) an legacy client wants to consume from Kinesis
 * 	2) a current client wants to consumer from DynamoDB streams
 *
 * In the context of 1), the set of configurations needs to be translated to the corresponding
 * configurations in the Kinesis listShards API. In the mean time, keep these configs since
 * they are applicable in the context of 2), i.e., polling data from a DynamoDB stream.
 * </p>
 *
 * @param configProps original config properties.
 * @return backfilled config properties.
 */
public static Properties backfillConsumerKeys(Properties configProps) {
	HashMap<String, String> oldKeyToNewKeys = new HashMap<>();
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_BASE, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_MAX, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_MAX);
	oldKeyToNewKeys.put(ConsumerConfigConstants.STREAM_DESCRIBE_BACKOFF_EXPONENTIAL_CONSTANT, ConsumerConfigConstants.LIST_SHARDS_BACKOFF_EXPONENTIAL_CONSTANT);
	for (Map.Entry<String, String> entry : oldKeyToNewKeys.entrySet()) {
		String oldKey = entry.getKey();
		String newKey = entry.getValue();
		if (configProps.containsKey(oldKey)) {
			configProps.setProperty(newKey, configProps.getProperty(oldKey));
			// Do not remove the oldKey since they may be used in the context of talking to DynamoDB streams
		}
	}
	return configProps;
}
 

@Test
public void testUnixTimestampForValidateOptionDateProperty() {
	String unixTimestamp = "1459799926.480";

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP, unixTimestamp);

	try {
		KinesisConfigUtil.validateConsumerConfiguration(testConfig);
	} catch (Exception e) {
		e.printStackTrace();
		fail();
	}
}
 

@Test
public void testUnparsableDoubleForGetShardIteratorBackoffExponentialConstantInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for get shard iterator operation backoff exponential constant");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETITERATOR_BACKOFF_EXPONENTIAL_CONSTANT, "unparsableDouble");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableIntForGetShardIteratorRetriesInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum retry attempts for getShardIterator shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETITERATOR_RETRIES, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableLongForGetShardIteratorBackoffBaseMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for get shard iterator operation base backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETITERATOR_BACKOFF_BASE, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableDoubleForGetShardIteratorBackoffExponentialConstantInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for get shard iterator operation backoff exponential constant");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETITERATOR_BACKOFF_EXPONENTIAL_CONSTANT, "unparsableDouble");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableIntForGetShardIteratorRetriesInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum retry attempts for getShardIterator shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETITERATOR_RETRIES, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableLongForShardDiscoveryIntervalMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for shard discovery sleep interval in milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableLongForGetRecordsIntervalMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for getRecords sleep interval in milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_INTERVAL_MILLIS, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableLongForListShardsBackoffBaseMillisInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for list shards operation base backoff milliseconds");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_BACKOFF_BASE, "unparsableLong");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnparsableIntForGetRecordsRetriesInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid value given for maximum retry attempts for getRecords shard operation");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.SHARD_GETRECORDS_RETRIES, "unparsableInt");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 

@Test
public void testStreamInitPositionTypeSetToAtTimestampButNoInitTimestampSetInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Please set value for initial timestamp ('"
			+ ConsumerConfigConstants.STREAM_INITIAL_TIMESTAMP + "') when using AT_TIMESTAMP initial position.");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "AT_TIMESTAMP");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testUnrecognizableStreamInitPositionTypeInConfig() {
	exception.expect(IllegalArgumentException.class);
	exception.expectMessage("Invalid initial position in stream");

	Properties testConfig = TestUtils.getStandardProperties();
	testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
	testConfig.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "wrongInitPosition");

	KinesisConfigUtil.validateConsumerConfiguration(testConfig);
}
 

@Test
public void testGetRecordsRetry() throws Exception {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");

	final GetRecordsResult expectedResult = new GetRecordsResult();
	MutableInt retries = new MutableInt();
	final Throwable[] retriableExceptions = new Throwable[] {
		new AmazonKinesisException("mock"),
	};

	AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
	Mockito.when(mockClient.getRecords(any())).thenAnswer(new Answer<GetRecordsResult>() {
		@Override
		public GetRecordsResult answer(InvocationOnMock invocation) throws Throwable{
			if (retries.intValue() < retriableExceptions.length) {
				retries.increment();
				throw retriableExceptions[retries.intValue() - 1];
			}
			return expectedResult;
		}
	});

	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);

	GetRecordsResult result = kinesisProxy.getRecords("fakeShardIterator", 1);
	assertEquals(retriableExceptions.length, retries.intValue());
	assertEquals(expectedResult, result);
}
 

private KinesisProxy getProxy(AmazonKinesis awsKinesis) {
	Properties kinesisConsumerConfig = new Properties();
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
	kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "fake_accesskey");
	kinesisConsumerConfig.setProperty(
		ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "fake_secretkey");
	KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
	Whitebox.setInternalState(kinesisProxy, "kinesisClient", awsKinesis);

	return kinesisProxy;
}
 

private static FlinkKinesisFirehoseProducer<String> createFirehoseSinkFromStaticConfig() {
	/*
	 * com.amazonaws.services.kinesisanalytics.flink.connectors.config.ProducerConfigConstants
	 * lists of all of the properties that firehose sink can be configured with.
	 */

	Properties outputProperties = new Properties();
	outputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, region);

	FlinkKinesisFirehoseProducer<String> sink = new FlinkKinesisFirehoseProducer<>(outputStreamName, new SimpleStringSchema(), outputProperties);
	ProducerConfigConstants config = new ProducerConfigConstants();
	return sink;
}