org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSkipCorruptedRecord() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null, /* periodic watermark assigner */
		null, /* punctuated watermark assigner */
		new TestProcessingTimeService(),
		0);

	final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	fetcher.emitRecord(1L, partitionStateHolder, 1L);
	fetcher.emitRecord(2L, partitionStateHolder, 2L);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(2L, partitionStateHolder.getOffset());

	// emit null record
	fetcher.emitRecord(null, partitionStateHolder, 3L);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue()); // the null record should be skipped
	assertEquals(3L, partitionStateHolder.getOffset()); // the offset in state still should have advanced
}
 
Example #2
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SafeVarargs
private MockFetcher(HashMap<KafkaTopicPartition, Long>... stateSnapshotsToReturn) throws Exception {
	super(
			new TestSourceContext<>(),
			new HashMap<>(),
			null /* watermark strategy */,
			new TestProcessingTimeService(),
			0,
			MockFetcher.class.getClassLoader(),
			new UnregisteredMetricsGroup(),
			false);

	this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn));
}
 
Example #3
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testFailingConsumerLifecycle(FlinkKafkaConsumerBase<String> testKafkaConsumer, @Nonnull Exception expectedException) throws Exception {
	try {
		setupConsumer(testKafkaConsumer);
		testKafkaConsumer.run(new TestSourceContext<>());

		fail("Exception should have been thrown from open / run method of FlinkKafkaConsumerBase.");
	} catch (Exception e) {
		assertThat(ExceptionUtils.findThrowable(e, throwable -> throwable.equals(expectedException)).isPresent(), is(true));
	}
	testKafkaConsumer.close();
}
 
Example #4
Source File: AbstractFetcherWatermarksTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<>(testWmStrategy),
			processingTimeProvider,
			10);

	processingTimeProvider.setCurrentTime(10);
	// no partitions; when the periodic watermark emitter fires, no watermark should be emitted
	assertFalse(sourceContext.hasWatermark());

	// counter-test that when the fetcher does actually have partitions,
	// when the periodic watermark emitter fires again, a watermark really is emitted
	fetcher.addDiscoveredPartitions(Collections.singletonList(
			new KafkaTopicPartition(testTopic, 0)));
	emitRecord(fetcher, 100L, fetcher.subscribedPartitionStates().get(0), 3L);
	processingTimeProvider.setCurrentTime(20);
	assertEquals(100, sourceContext.getLatestWatermark().getTimestamp());
}
 
Example #5
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
	// test data
	final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);

	// ----- create the test fetcher -----

	SourceContext<String> sourceContext = new TestSourceContext<>();
	Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets =
		Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);

	final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
	final OneShotLatch stateIterationBlockLatch = new OneShotLatch();

	final TestFetcher<String> fetcher = new TestFetcher<>(
		sourceContext,
		partitionsWithInitialOffsets,
		null, /* watermark strategy */
		new TestProcessingTimeService(),
		10,
		fetchLoopWaitLatch,
		stateIterationBlockLatch);

	// ----- run the fetcher -----

	final CheckedThread checkedThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			fetcher.runFetchLoop();
		}
	};
	checkedThread.start();

	// wait until state iteration begins before adding discovered partitions
	fetchLoopWaitLatch.await();
	fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));

	stateIterationBlockLatch.trigger();
	checkedThread.sync();
}
 
Example #6
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSkipCorruptedRecord() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			null, /* watermark strategy */
			new TestProcessingTimeService(),
			0);

	final KafkaTopicPartitionState<Long, Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	emitRecord(fetcher, 1L, partitionStateHolder, 1L);
	emitRecord(fetcher, 2L, partitionStateHolder, 2L);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(2L, partitionStateHolder.getOffset());

	// emit no records
	fetcher.emitRecordsWithTimestamps(emptyQueue(), partitionStateHolder, 3L, Long.MIN_VALUE);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue()); // the null record should be skipped
	assertEquals(3L, partitionStateHolder.getOffset()); // the offset in state still should have advanced
}
 
Example #7
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIgnorePartitionStateSentinelInSnapshot() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			null, /* watermark strategy */
			new TestProcessingTimeService(),
			0);

	synchronized (sourceContext.getCheckpointLock()) {
		HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState();
		fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() {
			@Override
			public void onSuccess() {
			}

			@Override
			public void onException(Throwable cause) {
				throw new RuntimeException("Callback failed", cause);
			}
		});

		assertTrue(fetcher.getLastCommittedOffsets().isPresent());
		assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get());
	}
}
 
Example #8
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SafeVarargs
private MockFetcher(HashMap<KafkaTopicPartition, Long>... stateSnapshotsToReturn) throws Exception {
	super(
			new TestSourceContext<>(),
			new HashMap<>(),
			null,
			null,
			new TestProcessingTimeService(),
			0,
			MockFetcher.class.getClassLoader(),
			new UnregisteredMetricsGroup(),
			false);

	this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn));
}
 
Example #9
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testFailingConsumerLifecycle(FlinkKafkaConsumerBase<String> testKafkaConsumer, @Nonnull Exception expectedException) throws Exception {
	try {
		setupConsumer(testKafkaConsumer);
		testKafkaConsumer.run(new TestSourceContext<>());

		fail("Exception should have been thrown from open / run method of FlinkKafkaConsumerBase.");
	} catch (Exception e) {
		assertThat(ExceptionUtils.findThrowable(e, throwable -> throwable.equals(expectedException)).isPresent(), is(true));
	}
	testKafkaConsumer.close();
}
 
Example #10
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()),
		null, /* punctuated watermarks assigner*/
		processingTimeProvider,
		10);

	processingTimeProvider.setCurrentTime(10);
	// no partitions; when the periodic watermark emitter fires, no watermark should be emitted
	assertFalse(sourceContext.hasWatermark());

	// counter-test that when the fetcher does actually have partitions,
	// when the periodic watermark emitter fires again, a watermark really is emitted
	fetcher.addDiscoveredPartitions(Collections.singletonList(new KafkaTopicPartition(testTopic, 0)));
	fetcher.emitRecord(100L, fetcher.subscribedPartitionStates().get(0), 3L);
	processingTimeProvider.setCurrentTime(20);
	assertEquals(100, sourceContext.getLatestWatermark().getTimestamp());
}
 
Example #11
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSkipCorruptedRecordWithPunctuatedWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null, /* periodic watermark assigner */
		new SerializedValue<AssignerWithPunctuatedWatermarks<Long>>(new PunctuatedTestExtractor()), /* punctuated watermark assigner */
		processingTimeProvider,
		0);

	final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	// elements generate a watermark if the timestamp is a multiple of three
	fetcher.emitRecord(1L, partitionStateHolder, 1L);
	fetcher.emitRecord(2L, partitionStateHolder, 2L);
	fetcher.emitRecord(3L, partitionStateHolder, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
	assertEquals(3L, partitionStateHolder.getOffset());

	// emit null record
	fetcher.emitRecord(null, partitionStateHolder, 4L);

	// no elements or watermarks should have been collected
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());
	// the offset in state still should have advanced
	assertEquals(4L, partitionStateHolder.getOffset());
}
 
Example #12
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSkipCorruptedRecord() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null, /* periodic watermark assigner */
		null, /* punctuated watermark assigner */
		new TestProcessingTimeService(),
		0);

	final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	fetcher.emitRecord(1L, partitionStateHolder, 1L);
	fetcher.emitRecord(2L, partitionStateHolder, 2L);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(2L, partitionStateHolder.getOffset());

	// emit null record
	fetcher.emitRecord(null, partitionStateHolder, 3L);
	assertEquals(2L, sourceContext.getLatestElement().getValue().longValue()); // the null record should be skipped
	assertEquals(3L, partitionStateHolder.getOffset()); // the offset in state still should have advanced
}
 
Example #13
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()),
		null, /* punctuated watermarks assigner*/
		processingTimeProvider,
		10);

	processingTimeProvider.setCurrentTime(10);
	// no partitions; when the periodic watermark emitter fires, no watermark should be emitted
	assertFalse(sourceContext.hasWatermark());

	// counter-test that when the fetcher does actually have partitions,
	// when the periodic watermark emitter fires again, a watermark really is emitted
	fetcher.addDiscoveredPartitions(Collections.singletonList(new KafkaTopicPartition(testTopic, 0)));
	fetcher.emitRecord(100L, fetcher.subscribedPartitionStates().get(0), 3L);
	processingTimeProvider.setCurrentTime(20);
	assertEquals(100, sourceContext.getLatestWatermark().getTimestamp());
}
 
Example #14
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testIgnorePartitionStateSentinelInSnapshot() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null,
		null,
		new TestProcessingTimeService(),
		0);

	synchronized (sourceContext.getCheckpointLock()) {
		HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState();
		fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() {
			@Override
			public void onSuccess() {
			}

			@Override
			public void onException(Throwable cause) {
				throw new RuntimeException("Callback failed", cause);
			}
		});

		assertTrue(fetcher.getLastCommittedOffsets().isPresent());
		assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get());
	}
}
 
Example #15
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSkipCorruptedRecordWithPunctuatedWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null, /* periodic watermark assigner */
		new SerializedValue<AssignerWithPunctuatedWatermarks<Long>>(new PunctuatedTestExtractor()), /* punctuated watermark assigner */
		processingTimeProvider,
		0);

	final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	// elements generate a watermark if the timestamp is a multiple of three
	fetcher.emitRecord(1L, partitionStateHolder, 1L);
	fetcher.emitRecord(2L, partitionStateHolder, 2L);
	fetcher.emitRecord(3L, partitionStateHolder, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
	assertEquals(3L, partitionStateHolder.getOffset());

	// emit null record
	fetcher.emitRecord(null, partitionStateHolder, 4L);

	// no elements or watermarks should have been collected
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());
	// the offset in state still should have advanced
	assertEquals(4L, partitionStateHolder.getOffset());
}
 
Example #16
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIgnorePartitionStateSentinelInSnapshot() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		null,
		null,
		new TestProcessingTimeService(),
		0);

	synchronized (sourceContext.getCheckpointLock()) {
		HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState();
		fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() {
			@Override
			public void onSuccess() {
			}

			@Override
			public void onException(Throwable cause) {
				throw new RuntimeException("Callback failed", cause);
			}
		});

		assertTrue(fetcher.getLastCommittedOffsets().isPresent());
		assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get());
	}
}
 
Example #17
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void testFailingConsumerLifecycle(FlinkKafkaConsumerBase<String> testKafkaConsumer, @Nonnull Exception expectedException) throws Exception {
	try {
		setupConsumer(testKafkaConsumer);
		testKafkaConsumer.run(new TestSourceContext<>());

		fail("Exception should have been thrown from open / run method of FlinkKafkaConsumerBase.");
	} catch (Exception e) {
		assertThat(ExceptionUtils.findThrowable(e, throwable -> throwable.equals(expectedException)).isPresent(), is(true));
	}
	testKafkaConsumer.close();
}
 
Example #18
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SafeVarargs
private MockFetcher(HashMap<KafkaTopicPartition, Long>... stateSnapshotsToReturn) throws Exception {
	super(
			new TestSourceContext<>(),
			new HashMap<>(),
			null,
			null,
			new TestProcessingTimeService(),
			0,
			MockFetcher.class.getClassLoader(),
			new UnregisteredMetricsGroup(),
			false);

	this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn));
}
 
Example #19
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void testNormalConsumerLifecycle(FlinkKafkaConsumerBase<String> testKafkaConsumer) throws Exception {
	setupConsumer(testKafkaConsumer);
	final CompletableFuture<Void> runFuture = CompletableFuture.runAsync(ThrowingRunnable.unchecked(() -> testKafkaConsumer.run(new TestSourceContext<>())));
	testKafkaConsumer.close();
	runFuture.get();
}
 
Example #20
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
	// test data
	final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);

	// ----- create the test fetcher -----

	@SuppressWarnings("unchecked")
	SourceContext<String> sourceContext = new TestSourceContext<>();
	Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets =
		Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);

	final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
	final OneShotLatch stateIterationBlockLatch = new OneShotLatch();

	final TestFetcher<String> fetcher = new TestFetcher<>(
		sourceContext,
		partitionsWithInitialOffsets,
		null, /* periodic assigner */
		null, /* punctuated assigner */
		new TestProcessingTimeService(),
		10,
		fetchLoopWaitLatch,
		stateIterationBlockLatch);

	// ----- run the fetcher -----

	final CheckedThread checkedThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			fetcher.runFetchLoop();
		}
	};
	checkedThread.start();

	// wait until state iteration begins before adding discovered partitions
	fetchLoopWaitLatch.await();
	fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));

	stateIterationBlockLatch.trigger();
	checkedThread.sync();
}
 
Example #21
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testPeriodicWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()),
			null, /* punctuated watermarks assigner*/
			processingTimeService,
			10);

	final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates().get(0);
	final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates().get(1);
	final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates().get(2);

	// elements generate a watermark if the timestamp is a multiple of three

	// elements for partition 1
	fetcher.emitRecord(1L, part1, 1L);
	fetcher.emitRecord(2L, part1, 2L);
	fetcher.emitRecord(3L, part1, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 2
	fetcher.emitRecord(12L, part2, 1L);
	assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(12L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 3
	fetcher.emitRecord(101L, part3, 1L);
	fetcher.emitRecord(102L, part3, 2L);
	assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(102L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(10);

	// now, we should have a watermark (this blocks until the periodic thread emitted the watermark)
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 3
	fetcher.emitRecord(1003L, part3, 3L);
	fetcher.emitRecord(1004L, part3, 4L);
	fetcher.emitRecord(1005L, part3, 5L);
	assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());

	// advance partition 1 beyond partition 2 - this bumps the watermark
	fetcher.emitRecord(30L, part1, 4L);
	assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(30L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(20);

	// this blocks until the periodic thread emitted the watermark
	assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 2 again - this bumps the watermark
	fetcher.emitRecord(13L, part2, 2L);
	fetcher.emitRecord(14L, part2, 3L);
	fetcher.emitRecord(15L, part2, 3L);

	processingTimeService.setCurrentTime(30);
	// this blocks until the periodic thread emitted the watermark
	long watermarkTs = sourceContext.getLatestWatermark().getTimestamp();
	assertTrue(watermarkTs >= 13L && watermarkTs <= 15L);
}
 
Example #22
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private void testNormalConsumerLifecycle(FlinkKafkaConsumerBase<String> testKafkaConsumer) throws Exception {
	setupConsumer(testKafkaConsumer);
	final CompletableFuture<Void> runFuture = CompletableFuture.runAsync(ThrowingRunnable.unchecked(() -> testKafkaConsumer.run(new TestSourceContext<>())));
	testKafkaConsumer.close();
	runFuture.get();
}
 
Example #23
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
	// test data
	final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);

	// ----- create the test fetcher -----

	@SuppressWarnings("unchecked")
	SourceContext<String> sourceContext = new TestSourceContext<>();
	Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets =
		Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);

	final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
	final OneShotLatch stateIterationBlockLatch = new OneShotLatch();

	final TestFetcher<String> fetcher = new TestFetcher<>(
		sourceContext,
		partitionsWithInitialOffsets,
		null, /* periodic assigner */
		null, /* punctuated assigner */
		new TestProcessingTimeService(),
		10,
		fetchLoopWaitLatch,
		stateIterationBlockLatch);

	// ----- run the fetcher -----

	final CheckedThread checkedThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			fetcher.runFetchLoop();
		}
	};
	checkedThread.start();

	// wait until state iteration begins before adding discovered partitions
	fetchLoopWaitLatch.await();
	fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));

	stateIterationBlockLatch.trigger();
	checkedThread.sync();
}
 
Example #24
Source File: AbstractFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testPunctuatedWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			null, /* periodic watermark assigner */
			new SerializedValue<AssignerWithPunctuatedWatermarks<Long>>(new PunctuatedTestExtractor()),
			processingTimeProvider,
			0);

	final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates().get(0);
	final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates().get(1);
	final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates().get(2);

	// elements generate a watermark if the timestamp is a multiple of three

	// elements for partition 1
	fetcher.emitRecord(1L, part1, 1L);
	fetcher.emitRecord(2L, part1, 2L);
	fetcher.emitRecord(3L, part1, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());

	// elements for partition 2
	fetcher.emitRecord(12L, part2, 1L);
	assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(12L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());

	// elements for partition 3
	fetcher.emitRecord(101L, part3, 1L);
	fetcher.emitRecord(102L, part3, 2L);
	assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(102L, sourceContext.getLatestElement().getTimestamp());

	// now, we should have a watermark
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 3
	fetcher.emitRecord(1003L, part3, 3L);
	fetcher.emitRecord(1004L, part3, 4L);
	fetcher.emitRecord(1005L, part3, 5L);
	assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());

	// advance partition 1 beyond partition 2 - this bumps the watermark
	fetcher.emitRecord(30L, part1, 4L);
	assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(30L, sourceContext.getLatestElement().getTimestamp());
	assertTrue(sourceContext.hasWatermark());
	assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 2 again - this bumps the watermark
	fetcher.emitRecord(13L, part2, 2L);
	assertFalse(sourceContext.hasWatermark());
	fetcher.emitRecord(14L, part2, 3L);
	assertFalse(sourceContext.hasWatermark());
	fetcher.emitRecord(15L, part2, 3L);
	assertTrue(sourceContext.hasWatermark());
	assertEquals(15L, sourceContext.getLatestWatermark().getTimestamp());
}
 
Example #25
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testPeriodicWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()),
			null, /* punctuated watermarks assigner*/
			processingTimeService,
			10);

	final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates().get(0);
	final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates().get(1);
	final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates().get(2);

	// elements generate a watermark if the timestamp is a multiple of three

	// elements for partition 1
	fetcher.emitRecord(1L, part1, 1L);
	fetcher.emitRecord(2L, part1, 2L);
	fetcher.emitRecord(3L, part1, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 2
	fetcher.emitRecord(12L, part2, 1L);
	assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(12L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 3
	fetcher.emitRecord(101L, part3, 1L);
	fetcher.emitRecord(102L, part3, 2L);
	assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(102L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(10);

	// now, we should have a watermark (this blocks until the periodic thread emitted the watermark)
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 3
	fetcher.emitRecord(1003L, part3, 3L);
	fetcher.emitRecord(1004L, part3, 4L);
	fetcher.emitRecord(1005L, part3, 5L);
	assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());

	// advance partition 1 beyond partition 2 - this bumps the watermark
	fetcher.emitRecord(30L, part1, 4L);
	assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(30L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(20);

	// this blocks until the periodic thread emitted the watermark
	assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 2 again - this bumps the watermark
	fetcher.emitRecord(13L, part2, 2L);
	fetcher.emitRecord(14L, part2, 3L);
	fetcher.emitRecord(15L, part2, 3L);

	processingTimeService.setCurrentTime(30);
	// this blocks until the periodic thread emitted the watermark
	long watermarkTs = sourceContext.getLatestWatermark().getTimestamp();
	assertTrue(watermarkTs >= 13L && watermarkTs <= 15L);
}
 
Example #26
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testPunctuatedWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 7), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 13), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(new KafkaTopicPartition(testTopic, 21), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			null, /* periodic watermark assigner */
			new SerializedValue<AssignerWithPunctuatedWatermarks<Long>>(new PunctuatedTestExtractor()),
			processingTimeProvider,
			0);

	final KafkaTopicPartitionState<Object> part1 = fetcher.subscribedPartitionStates().get(0);
	final KafkaTopicPartitionState<Object> part2 = fetcher.subscribedPartitionStates().get(1);
	final KafkaTopicPartitionState<Object> part3 = fetcher.subscribedPartitionStates().get(2);

	// elements generate a watermark if the timestamp is a multiple of three

	// elements for partition 1
	fetcher.emitRecord(1L, part1, 1L);
	fetcher.emitRecord(2L, part1, 2L);
	fetcher.emitRecord(3L, part1, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());

	// elements for partition 2
	fetcher.emitRecord(12L, part2, 1L);
	assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(12L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());

	// elements for partition 3
	fetcher.emitRecord(101L, part3, 1L);
	fetcher.emitRecord(102L, part3, 2L);
	assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(102L, sourceContext.getLatestElement().getTimestamp());

	// now, we should have a watermark
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 3
	fetcher.emitRecord(1003L, part3, 3L);
	fetcher.emitRecord(1004L, part3, 4L);
	fetcher.emitRecord(1005L, part3, 5L);
	assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());

	// advance partition 1 beyond partition 2 - this bumps the watermark
	fetcher.emitRecord(30L, part1, 4L);
	assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(30L, sourceContext.getLatestElement().getTimestamp());
	assertTrue(sourceContext.hasWatermark());
	assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 2 again - this bumps the watermark
	fetcher.emitRecord(13L, part2, 2L);
	assertFalse(sourceContext.hasWatermark());
	fetcher.emitRecord(14L, part2, 3L);
	assertFalse(sourceContext.hasWatermark());
	fetcher.emitRecord(15L, part2, 3L);
	assertTrue(sourceContext.hasWatermark());
	assertEquals(15L, sourceContext.getLatestWatermark().getTimestamp());
}
 
Example #27
Source File: AbstractFetcherWatermarksTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testPeriodicWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(
			new KafkaTopicPartition(testTopic, 7),
			KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(
			new KafkaTopicPartition(testTopic, 13),
			KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
	originalPartitions.put(
			new KafkaTopicPartition(testTopic, 21),
			KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeService = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<>(testWmStrategy),
			processingTimeService,
			10);

	final KafkaTopicPartitionState<Long, Object> part1 =
			fetcher.subscribedPartitionStates().get(0);
	final KafkaTopicPartitionState<Long, Object> part2 =
			fetcher.subscribedPartitionStates().get(1);
	final KafkaTopicPartitionState<Long, Object> part3 =
			fetcher.subscribedPartitionStates().get(2);

	// elements generate a watermark if the timestamp is a multiple of three

	// elements for partition 1
	emitRecord(fetcher, 1L, part1, 1L);
	emitRecord(fetcher, 1L, part1, 1L);
	emitRecord(fetcher, 2L, part1, 2L);
	emitRecord(fetcher, 3L, part1, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 2
	emitRecord(fetcher, 12L, part2, 1L);
	assertEquals(12L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(12L, sourceContext.getLatestElement().getTimestamp());

	// elements for partition 3
	emitRecord(fetcher, 101L, part3, 1L);
	emitRecord(fetcher, 102L, part3, 2L);
	assertEquals(102L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(102L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(10);

	// now, we should have a watermark (this blocks until the periodic thread emitted the watermark)
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 3
	emitRecord(fetcher, 1003L, part3, 3L);
	emitRecord(fetcher, 1004L, part3, 4L);
	emitRecord(fetcher, 1005L, part3, 5L);
	assertEquals(1005L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(1005L, sourceContext.getLatestElement().getTimestamp());

	// advance partition 1 beyond partition 2 - this bumps the watermark
	emitRecord(fetcher, 30L, part1, 4L);
	assertEquals(30L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(30L, sourceContext.getLatestElement().getTimestamp());

	processingTimeService.setCurrentTime(20);

	// this blocks until the periodic thread emitted the watermark
	assertEquals(12L, sourceContext.getLatestWatermark().getTimestamp());

	// advance partition 2 again - this bumps the watermark
	emitRecord(fetcher, 13L, part2, 2L);
	emitRecord(fetcher, 14L, part2, 3L);
	emitRecord(fetcher, 15L, part2, 3L);

	processingTimeService.setCurrentTime(30);
	// this blocks until the periodic thread emitted the watermark
	long watermarkTs = sourceContext.getLatestWatermark().getTimestamp();
	assertTrue(watermarkTs >= 13L && watermarkTs <= 15L);
}
 
Example #28
Source File: AbstractFetcherWatermarksTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSkipCorruptedRecordWithPeriodicWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(
			new KafkaTopicPartition(testTopic, 1),
			KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<>(testWmStrategy),
			processingTimeProvider,
			10);

	final KafkaTopicPartitionState<Long, Object> partitionStateHolder =
			fetcher.subscribedPartitionStates().get(0);

	// elements generate a watermark if the timestamp is a multiple of three
	emitRecord(fetcher, 1L, partitionStateHolder, 1L);
	emitRecord(fetcher, 2L, partitionStateHolder, 2L);
	emitRecord(fetcher, 3L, partitionStateHolder, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertEquals(3L, partitionStateHolder.getOffset());

	// advance timer for watermark emitting
	processingTimeProvider.setCurrentTime(10L);
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// emit no records
	fetcher.emitRecordsWithTimestamps(
			emptyQueue(),
			partitionStateHolder,
			4L,
			Long.MIN_VALUE);

	// no elements should have been collected
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	// the offset in state still should have advanced
	assertEquals(4L, partitionStateHolder.getOffset());

	// no watermarks should be collected
	processingTimeProvider.setCurrentTime(20L);
	assertFalse(sourceContext.hasWatermark());
}
 
Example #29
Source File: AbstractFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSkipCorruptedRecordWithPeriodicWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	TestFetcher<Long> fetcher = new TestFetcher<>(
		sourceContext,
		originalPartitions,
		new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), /* periodic watermark assigner */
		null, /* punctuated watermark assigner */
		processingTimeProvider,
		10);

	final KafkaTopicPartitionState<Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);

	// elements generate a watermark if the timestamp is a multiple of three
	fetcher.emitRecord(1L, partitionStateHolder, 1L);
	fetcher.emitRecord(2L, partitionStateHolder, 2L);
	fetcher.emitRecord(3L, partitionStateHolder, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertEquals(3L, partitionStateHolder.getOffset());

	// advance timer for watermark emitting
	processingTimeProvider.setCurrentTime(10L);
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());

	// emit null record
	fetcher.emitRecord(null, partitionStateHolder, 4L);

	// no elements should have been collected
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	// the offset in state still should have advanced
	assertEquals(4L, partitionStateHolder.getOffset());

	// no watermarks should be collected
	processingTimeProvider.setCurrentTime(20L);
	assertFalse(sourceContext.hasWatermark());
}
 
Example #30
Source File: AbstractFetcherWatermarksTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSkipCorruptedRecordWithPunctuatedWatermarks() throws Exception {
	final String testTopic = "test topic name";
	Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
	originalPartitions.put(
			new KafkaTopicPartition(testTopic, 1),
			KafkaTopicPartitionStateSentinel.LATEST_OFFSET);

	TestSourceContext<Long> sourceContext = new TestSourceContext<>();

	TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService();

	AssignerWithPunctuatedWatermarksAdapter.Strategy<Long> testWmStrategy =
			new AssignerWithPunctuatedWatermarksAdapter.Strategy<>(new PunctuatedTestExtractor());

	TestFetcher<Long> fetcher = new TestFetcher<>(
			sourceContext,
			originalPartitions,
			new SerializedValue<>(testWmStrategy),
			processingTimeProvider,
			0);

	final KafkaTopicPartitionState<Long, Object> partitionStateHolder =
			fetcher.subscribedPartitionStates().get(0);

	// elements generate a watermark if the timestamp is a multiple of three
	emitRecord(fetcher, 1L, partitionStateHolder, 1L);
	emitRecord(fetcher, 2L, partitionStateHolder, 2L);
	emitRecord(fetcher, 3L, partitionStateHolder, 3L);
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertTrue(sourceContext.hasWatermark());
	assertEquals(3L, sourceContext.getLatestWatermark().getTimestamp());
	assertEquals(3L, partitionStateHolder.getOffset());

	// emit no records
	fetcher.emitRecordsWithTimestamps(emptyQueue(), partitionStateHolder, 4L, -1L);

	// no elements or watermarks should have been collected
	assertEquals(3L, sourceContext.getLatestElement().getValue().longValue());
	assertEquals(3L, sourceContext.getLatestElement().getTimestamp());
	assertFalse(sourceContext.hasWatermark());
	// the offset in state still should have advanced
	assertEquals(4L, partitionStateHolder.getOffset());
}