Java Code Examples for org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 
Example 2
Source Project: Flink-CEPplus   Source File: KinesisPubsubClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 
Example 3
Source Project: flink   Source File: KinesisPubsubClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 
Example 4
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 6 votes vote down vote up
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 
Example 5
Source Project: flink   Source File: KinesisPubsubClient.java    License: Apache License 2.0 6 votes vote down vote up
public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 
Example 6
Source Project: Flink-CEPplus   Source File: ShardConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsReporting() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(
			KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard,
			new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter();
	long millisBehindLatest = 500L;
	new ShardConsumer<>(
		fetcher,
		0,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest),
		shardMetricsReporter).run();

	// the millisBehindLatest metric should have been reported
	assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest());
}
 
Example 7
Source Project: Flink-CEPplus   Source File: ShardConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedState() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L),
		new ShardMetricsReporter()).run();

	assertEquals(1000, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 8
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
		final int numOfRecords,
		final int numOfGetRecordsCall,
		final int orderOfCallToExpire,
		final long millisBehindLatest) {
	return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
		numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
 
Example 9
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
		final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
			millisBehindLatest);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: TestableKinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
public TestableKinesisDataFetcher(
		List<String> fakeStreams,
		SourceFunction.SourceContext<T> sourceContext,
		Properties fakeConfiguration,
		KinesisDeserializationSchema<T> deserializationSchema,
		int fakeTotalCountOfSubtasks,
		int fakeIndexOfThisSubtask,
		AtomicReference<Throwable> thrownErrorUnderTest,
		LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		KinesisProxyInterface fakeKinesis) {
	super(
		fakeStreams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
		fakeConfiguration,
		deserializationSchema,
		DEFAULT_SHARD_ASSIGNER,
		null,
		null,
		thrownErrorUnderTest,
		subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		(properties) -> fakeKinesis);

	this.runWaiter = new OneShotLatch();
	this.initialDiscoveryWaiter = new OneShotLatch();
	this.shutdownWaiter = new OneShotLatch();

	this.running = true;
}
 
Example 11
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
		final SourceFunction.SourceContext<T> sourceContext,
		final Properties fakeConfiguration,
		final KinesisDeserializationSchema<T> deserializationSchema,
		final int fakeTotalCountOfSubtasks,
		final int fakeIndexOfThisSubtask,
		final AtomicReference<Throwable> thrownErrorUnderTest,
		final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		final KinesisProxyInterface fakeKinesis) {
	super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
		fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
 
Example 12
Source Project: flink   Source File: ShardConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsReporting() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(
			KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard,
			new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter();
	long millisBehindLatest = 500L;
	new ShardConsumer<>(
		fetcher,
		0,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest),
		shardMetricsReporter).run();

	// the millisBehindLatest metric should have been reported
	assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest());
}
 
Example 13
Source Project: flink   Source File: ShardConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedState() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L),
		new ShardMetricsReporter()).run();

	assertEquals(1000, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 14
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 
Example 15
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
		final int numOfRecords,
		final int numOfGetRecordsCall,
		final int orderOfCallToExpire,
		final long millisBehindLatest) {
	return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
		numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
 
Example 16
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
		final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
			millisBehindLatest);
}
 
Example 17
Source Project: flink   Source File: TestableKinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
public TestableKinesisDataFetcher(
		List<String> fakeStreams,
		SourceFunction.SourceContext<T> sourceContext,
		Properties fakeConfiguration,
		KinesisDeserializationSchema<T> deserializationSchema,
		int fakeTotalCountOfSubtasks,
		int fakeIndexOfThisSubtask,
		AtomicReference<Throwable> thrownErrorUnderTest,
		LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		KinesisProxyInterface fakeKinesis) {
	super(
		fakeStreams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
		fakeConfiguration,
		deserializationSchema,
		DEFAULT_SHARD_ASSIGNER,
		null,
		null,
		thrownErrorUnderTest,
		subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		(properties) -> fakeKinesis);

	this.runWaiter = new OneShotLatch();
	this.initialDiscoveryWaiter = new OneShotLatch();
	this.shutdownWaiter = new OneShotLatch();

	this.running = true;
}
 
Example 18
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
		final SourceFunction.SourceContext<T> sourceContext,
		final Properties fakeConfiguration,
		final KinesisDeserializationSchema<T> deserializationSchema,
		final int fakeTotalCountOfSubtasks,
		final int fakeIndexOfThisSubtask,
		final AtomicReference<Throwable> thrownErrorUnderTest,
		final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		final KinesisProxyInterface fakeKinesis) {
	super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
		fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
 
Example 19
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
		final int numOfRecords,
		final int numOfGetRecordsCall,
		final int orderOfCallToExpire,
		final long millisBehindLatest) {
	return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
		numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
 
Example 20
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
		final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
			millisBehindLatest);
}
 
Example 21
Source Project: flink   Source File: TestableKinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
public TestableKinesisDataFetcher(
		List<String> fakeStreams,
		SourceFunction.SourceContext<T> sourceContext,
		Properties fakeConfiguration,
		KinesisDeserializationSchema<T> deserializationSchema,
		int fakeTotalCountOfSubtasks,
		int fakeIndexOfThisSubtask,
		AtomicReference<Throwable> thrownErrorUnderTest,
		LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		KinesisProxyInterface fakeKinesis) {
	super(
		fakeStreams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
		fakeConfiguration,
		deserializationSchema,
		DEFAULT_SHARD_ASSIGNER,
		null,
		null,
		thrownErrorUnderTest,
		subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		(properties) -> fakeKinesis);

	this.runWaiter = new OneShotLatch();
	this.initialDiscoveryWaiter = new OneShotLatch();
	this.shutdownWaiter = new OneShotLatch();

	this.running = true;
}
 
Example 22
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
		final SourceFunction.SourceContext<T> sourceContext,
		final Properties fakeConfiguration,
		final KinesisDeserializationSchema<T> deserializationSchema,
		final int fakeTotalCountOfSubtasks,
		final int fakeIndexOfThisSubtask,
		final AtomicReference<Throwable> thrownErrorUnderTest,
		final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		final KinesisProxyInterface fakeKinesis) {
	super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
		fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ShardConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		// Get a total of 1000 records with 9 getRecords() calls,
		// and the 7th getRecords() call will encounter an unexpected expired shard iterator
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
			1000, 9, 7, 500L),
		new ShardMetricsReporter()).run();

	assertEquals(1000, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 24
Source Project: Flink-CEPplus   Source File: ShardConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() {
	Properties consumerProperties = new Properties();
	consumerProperties.put("flink.shard.adaptivereads", "true");

	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			consumerProperties,
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		// Initial number of records to fetch --> 10
		FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L),
		new ShardMetricsReporter()).run();

	// Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb
	// Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40
	// Total number of records = 10 + 40 = 50
	assertEquals(50, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 25
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCalls(
		final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	return new SingleShardEmittingFixNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls, millisBehindLatest);
}
 
Example 26
public static KinesisProxyInterface blockingQueueGetRecords(Map<String, List<BlockingQueue<String>>> streamsToShardQueues) {
	return new BlockingQueueKinesis(streamsToShardQueues);
}
 
Example 27
Source Project: flink   Source File: ShardConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() {
	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			new Properties(),
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		// Get a total of 1000 records with 9 getRecords() calls,
		// and the 7th getRecords() call will encounter an unexpected expired shard iterator
		FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
			1000, 9, 7, 500L),
		new ShardMetricsReporter()).run();

	assertEquals(1000, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 28
Source Project: flink   Source File: ShardConsumerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() {
	Properties consumerProperties = new Properties();
	consumerProperties.put("flink.shard.adaptivereads", "true");

	StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);

	LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
	subscribedShardsStateUnderTest.add(
		new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
			fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));

	TestSourceContext<String> sourceContext = new TestSourceContext<>();

	TestableKinesisDataFetcher<String> fetcher =
		new TestableKinesisDataFetcher<>(
			Collections.singletonList("fakeStream"),
			sourceContext,
			consumerProperties,
			new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
			10,
			2,
			new AtomicReference<>(),
			subscribedShardsStateUnderTest,
			KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
			Mockito.mock(KinesisProxyInterface.class));

	int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
	new ShardConsumer<>(
		fetcher,
		shardIndex,
		subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
		// Initial number of records to fetch --> 10
		FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L),
		new ShardMetricsReporter()).run();

	// Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb
	// Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40
	// Total number of records = 10 + 40 = 50
	assertEquals(50, sourceContext.getCollectedOutputs().size());
	assertEquals(
		SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
		subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
 
Example 29
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCalls(
		final int numOfRecords,
		final int numOfGetRecordsCalls,
		final long millisBehindLatest) {
	return new SingleShardEmittingFixNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls, millisBehindLatest);
}
 
Example 30
Source Project: flink   Source File: FakeKinesisBehavioursFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static KinesisProxyInterface blockingQueueGetRecords(Map<String, List<BlockingQueue<String>>> streamsToShardQueues) {
	return new BlockingQueueKinesis(streamsToShardQueues);
}