Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#processElement()

The following examples show how to use org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#processElement() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: TopNFunctionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOutputRankNumberWithVariableRankRange() throws Exception {
	AbstractTopNFunction func = createFunction(RankType.ROW_NUMBER, new VariableRankRange(1), true, true);
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(func);
	testHarness.open();
	testHarness.processElement(insertRecord("book", 2L, 12));
	testHarness.processElement(insertRecord("book", 2L, 19));
	testHarness.processElement(insertRecord("book", 2L, 11));
	testHarness.processElement(insertRecord("fruit", 1L, 33));
	testHarness.processElement(insertRecord("fruit", 1L, 44));
	testHarness.processElement(insertRecord("fruit", 1L, 22));
	testHarness.close();

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord("book", 2L, 12, 1L));
	expectedOutput.add(insertRecord("book", 2L, 19, 2L));
	expectedOutput.add(updateBeforeRecord("book", 2L, 12, 1L));
	expectedOutput.add(updateAfterRecord("book", 2L, 11, 1L));
	expectedOutput.add(updateBeforeRecord("book", 2L, 19, 2L));
	expectedOutput.add(updateAfterRecord("book", 2L, 12, 2L));
	expectedOutput.add(insertRecord("fruit", 1L, 33, 1L));
	expectedOutput.add(updateBeforeRecord("fruit", 1L, 33, 1L));
	expectedOutput.add(updateAfterRecord("fruit", 1L, 22, 1L));
	assertorWithRowNumber
			.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 2
Source Project: flink   File: FlinkKafkaProducerITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 
Example 3
@Test
public void testFinishBundleTriggeredByTime() throws Exception {
	Configuration conf = new Configuration();
	conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10);
	conf.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, 1000L);
	OneInputStreamOperatorTestHarness<IN, OUT> testHarness = getTestHarness(conf, JoinRelType.INNER);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1));
	assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput());

	testHarness.setProcessingTime(1000L);
	expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 0L)));
	assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 4
@Test
public void periodicWatermarksDoNotRegress() throws Exception {
	OneInputStreamOperatorTestHarness<Long, Long> testHarness = createTestHarness(
			WatermarkStrategy
					.forGenerator((ctx) -> new PeriodicWatermarkGenerator())
					.withTimestampAssigner((ctx) -> new LongExtractor()));

	testHarness.processElement(new StreamRecord<>(4L, 1));
	testHarness.setProcessingTime(AUTO_WATERMARK_INTERVAL);

	assertThat(pollNextStreamRecord(testHarness), streamRecord(4L, 4L));
	assertThat(pollNextLegacyWatermark(testHarness), is(legacyWatermark(3L)));

	testHarness.processElement(new StreamRecord<>(2L, 1));
	testHarness.setProcessingTime(AUTO_WATERMARK_INTERVAL);

	assertThat(pollNextStreamRecord(testHarness), streamRecord(2L, 2L));
	assertThat(testHarness.getOutput(), empty());
}
 
Example 5
Source Project: flink   File: KeyedProcessOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullOutputTagRefusal() throws Exception {
	KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(17);
	try {
		expectedException.expect(IllegalArgumentException.class);
		testHarness.processElement(new StreamRecord<>(5));
	} finally {
		testHarness.close();
	}
}
 
Example 6
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new UserExceptionAsyncFunction(), TIMEOUT, 2, outputMode);

	harness.getEnvironment().setExpectedExternalFailureCause(Throwable.class);
	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 
Example 7
/**
 * This also verifies that the timestamps ouf side-emitted records is correct.
 */
@Test
public void testSideOutput() throws Exception {
	KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new SideOutputProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>();
	expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedIntSideOutput,
		intSideOutput);

	ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>();
	expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */));
	ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput =
		testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG);
	TestHarnessUtil.assertOutputEquals(
		"Side output was not correct.",
		expectedLongSideOutput,
		longSideOutput);

	testHarness.close();
}
 
Example 8
/**
 * Test ensuring that if an invoke call happens right after an async exception is caught, it should be rethrown.
 */
@Test
public void testAsyncErrorRethrownOnInvoke() throws Throwable {
	final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>(
		FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), null);

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));

	testHarness.open();

	testHarness.processElement(new StreamRecord<>("msg-1"));

	// let the message request return an async exception
	producer.getPendingCallbacks().get(0).onCompletion(null, new Exception("artificial async exception"));

	try {
		testHarness.processElement(new StreamRecord<>("msg-2"));
	} catch (Exception e) {
		// the next invoke should rethrow the async exception
		Assert.assertTrue(e.getCause().getMessage().contains("artificial async exception"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 
Example 9
Source Project: flink   File: KeyedProcessOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7
	testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new StreamRecord<>("INPUT:13", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 10
/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(13)); // should delete timer
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:13"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 
Example 11
@Test
public void testTemporalInnerAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(record(1, "a"));
		testHarness.processElement(record(2, "b"));
		testHarness.processElement(record(3, "c"));
		testHarness.processElement(record(4, "d"));
		testHarness.processElement(record(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(1, "a", 1, "Julian"));
	expectedOutput.add(record(3, "c", 3, "Jackson"));
	expectedOutput.add(record(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 12
/**
 * Tests TimeEvictor evictBefore behavior.
 */
@Test
public void testTimeEvictorEvictBefore() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);
	final int triggerCount = 2;
	final int windowSize = 4;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
		(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
		new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
		CountTrigger.of(triggerCount),
		TimeEvictor.of(Time.seconds(2)),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 5999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1001));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 6500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1002));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 7999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 3999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.close();

	Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
 
Example 13
@Test
public void testRowTimeWatermarkAssigner() throws Exception {
	final RowTimeMiniBatchAssginerOperator operator = new RowTimeMiniBatchAssginerOperator(5);
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator);
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(GenericRowData.of(1L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(2L)));
	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(3L)));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(4L)));
	testHarness.processWatermark(new Watermark(3));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(5L)));
	testHarness.processWatermark(new Watermark(4));
	testHarness.processWatermark(new Watermark(5));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(7L)));
	testHarness.processWatermark(new Watermark(6));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(11L)));
	testHarness.processWatermark(new Watermark(10));
	testHarness.processWatermark(new Watermark(12));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(16L)));
	testHarness.processWatermark(new Watermark(15));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(17L)));
	testHarness.processWatermark(new Watermark(16));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(20L)));
	testHarness.processWatermark(new Watermark(19));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(22L)));
	testHarness.processWatermark(new Watermark(20));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(24L)));
	testHarness.processWatermark(new Watermark(21));
	testHarness.processElement(new StreamRecord<>(GenericRowData.of(25L)));

	testHarness.close();

	List<Watermark> expected = new ArrayList<>();
	expected.add(new Watermark(4));
	expected.add(new Watermark(10));
	expected.add(new Watermark(15));
	expected.add(new Watermark(19));
	expected.add(new Watermark(21)); // the last buffered watermark

	ConcurrentLinkedQueue<Object> output = testHarness.getOutput();
	List<Watermark> watermarks = extractWatermarks(output);
	assertEquals(expected, watermarks);
	// verify all the records are forwarded, there are 13 records.
	assertEquals(expected.size() + 13, output.size());
}
 
Example 14
Source Project: flink   File: WindowOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSlidingCountWindow() throws Exception {
	closeCalled.set(0);
	final int windowSize = 5;
	final int windowSlide = 3;
	LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };

	WindowOperator operator = WindowOperatorBuilder
			.builder()
			.withInputFields(inputFieldTypes)
			.countWindow(windowSize, windowSlide)
			.aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(insertRecord("key2", 1, 0L));
	testHarness.processElement(insertRecord("key2", 2, 1000L));
	testHarness.processElement(insertRecord("key2", 3, 2500L));
	testHarness.processElement(insertRecord("key2", 4, 2500L));
	testHarness.processElement(insertRecord("key2", 5, 2500L));
	testHarness.processElement(insertRecord("key1", 1, 10L));
	testHarness.processElement(insertRecord("key1", 2, 1000L));

	testHarness.processWatermark(new Watermark(12000));
	testHarness.setProcessingTime(12000L);
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 15L, 5L, 0L)));
	expectedOutput.add(new Watermark(12000));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
	testHarness.close();
	expectedOutput.clear();

	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshotV2);
	testHarness.open();

	testHarness.processElement(insertRecord("key1", 3, 2500L));
	testHarness.processElement(insertRecord("key1", 4, 2500L));
	testHarness.processElement(insertRecord("key1", 5, 2500L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 15L, 5L, 0L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key2", 6, 6000L));
	testHarness.processElement(insertRecord("key2", 7, 6000L));
	testHarness.processElement(insertRecord("key2", 8, 6050L));
	testHarness.processElement(insertRecord("key2", 9, 6050L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 30L, 5L, 1L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processElement(insertRecord("key1", 6, 4000L));
	testHarness.processElement(insertRecord("key1", 7, 4000L));
	testHarness.processElement(insertRecord("key1", 8, 4000L));
	testHarness.processElement(insertRecord("key2", 10, 15000L));
	testHarness.processElement(insertRecord("key2", 11, 15000L));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 30L, 5L, 1L)));
	expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 45L, 5L, 2L)));
	assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();

	// we close once in the rest...
	assertEquals("Close was not called.", 2, closeCalled.get());
}
 
Example 15
/**
 * Tests that any item failure in the listener callbacks due to flushing on an immediately following checkpoint
 * is rethrown; we set a timeout because the test will not finish if the logic is broken.
 */
@Test(timeout = 5000)
public void testItemFailureRethrownOnCheckpointAfterFlush() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and its mock item failures

	List<Exception> mockResponsesList = new ArrayList<>(2);
	mockResponsesList.add(null); // the first request in a bulk will succeed
	mockResponsesList.add(new Exception("artificial failure for record")); // the second request in a bulk will fail
	sink.setMockItemFailuresListForNextBulkItemResponses(mockResponsesList);

	testHarness.processElement(new StreamRecord<>("msg-1"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class));

	// manually execute the next bulk request (1 request only, thus should succeed)
	sink.manualBulkRequestWithAllPendingRequests();

	// setup the requests to be flushed in the snapshot
	testHarness.processElement(new StreamRecord<>("msg-2"));
	testHarness.processElement(new StreamRecord<>("msg-3"));
	verify(sink.getMockBulkProcessor(), times(3)).add(any(IndexRequest.class));

	CheckedThread snapshotThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			testHarness.snapshot(1L, 1000L);
		}
	};
	snapshotThread.start();

	// the snapshot should eventually be blocked before snapshot triggers flushing
	while (snapshotThread.getState() != Thread.State.WAITING) {
		Thread.sleep(10);
	}

	// let the snapshot-triggered flush continue (2 records in the bulk, so the 2nd one should fail)
	sink.continueFlush();

	try {
		snapshotThread.sync();
	} catch (Exception e) {
		// the snapshot should have failed with the failure from the 2nd request
		Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for record"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 
Example 16
Source Project: flink   File: GenericWriteAheadSinkTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
/**
 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
 * and later retry of the affected checkpoints.
 */
public void testCommitterException() throws Exception {

	ListSink2 sink = new ListSink2();

	OneInputStreamOperatorTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;

	for (int x = 0; x < 10; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(0, 0);
	testHarness.notifyOfCompletedCheckpoint(0);

	//isCommitted should have failed, thus sendValues() should never have been called
	Assert.assertEquals(0, sink.values.size());

	for (int x = 0; x < 11; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(1, 0);
	testHarness.notifyOfCompletedCheckpoint(1);

	//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
	Assert.assertEquals(10, sink.values.size());

	for (int x = 0; x < 12; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(2, 0);
	testHarness.notifyOfCompletedCheckpoint(2);

	//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 11 + 12 = 43 values
	Assert.assertEquals(43, sink.values.size());
}
 
Example 17
Source Project: flink   File: WindowOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests whether merging works correctly with the ContinuousEventTimeTrigger.
 */
@Test
public void testSessionWindowsWithContinuousEventTimeTrigger() throws Exception {
	closeCalled.set(0);

	final int sessionSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
		EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
		new TimeWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new SessionWindowFunction()),
		ContinuousEventTimeTrigger.of(Time.seconds(2)),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// add elements out-of-order and first trigger time is 2000
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 1500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));

	// triggers emit and next trigger time is 4000
	testHarness.processWatermark(new Watermark(2500));

	expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-1", 1500L, 4500L), 4499));
	expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499));
	expectedOutput.add(new Watermark(2500));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 4000));
	testHarness.processWatermark(new Watermark(3000));
	expectedOutput.add(new Watermark(3000));

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
	testHarness.close();

	expectedOutput.clear();
	testHarness = createTestHarness(operator);
	testHarness.setup();
	testHarness.initializeState(snapshot);
	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 4000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 3500));
	// triggers emit and next trigger time is 6000
	testHarness.processWatermark(new Watermark(4000));

	expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-3", 1500L, 7000L), 6999));
	expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-15", 0L, 7000L), 6999));
	expectedOutput.add(new Watermark(4000));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());

	testHarness.close();
}
 
Example 18
Source Project: flink   File: WindowOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testTumblingEventTimeWindows(OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator) throws Exception {
	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	// do a snapshot, close and restore again
	OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L);
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
	testHarness.close();

	testHarness = createTestHarness(operator);
	expectedOutput.clear();
	testHarness.setup();
	testHarness.initializeState(snapshot);
	testHarness.open();

	testHarness.processWatermark(new Watermark(2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
	expectedOutput.add(new Watermark(2999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processWatermark(new Watermark(3999));
	expectedOutput.add(new Watermark(3999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processWatermark(new Watermark(4999));
	expectedOutput.add(new Watermark(4999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processWatermark(new Watermark(5999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
	expectedOutput.add(new Watermark(5999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	// those don't have any effect...
	testHarness.processWatermark(new Watermark(6999));
	testHarness.processWatermark(new Watermark(7999));
	expectedOutput.add(new Watermark(6999));
	expectedOutput.add(new Watermark(7999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.close();
}
 
Example 19
@Test
public void testProcessingTimeTumblingWindows() throws Throwable {
	final int windowSize = 3;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
			new SumReducer(),
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
			ProcessingTimeTrigger.create(),
			0,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			createTestHarness(operator);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(3);

	// timestamp is ignored in processing time
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), Long.MAX_VALUE));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));

	testHarness.setProcessingTime(5000);

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), 2999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 7000));

	testHarness.setProcessingTime(7000);

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());

	testHarness.close();
}
 
Example 20
Source Project: flink   File: WindowOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSideOutputDueToLatenessSessionZeroLatenessPurgingTrigger() throws Exception {
	final int gapSize = 3;
	final long lateness = 0;

	ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer(),
		STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
			PurgingTrigger.of(EventTimeTrigger.create()),
			lateness,
			lateOutputTag);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
	ConcurrentLinkedQueue<Object> sideExpected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(1999));

	expected.add(new Watermark(1999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new Watermark(4998));

	// this will not be dropped because the session we're adding two has maxTimestamp
	// after the current watermark
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
	testHarness.processWatermark(new Watermark(7400));

	expected.add(new Watermark(7400));

	// this will merge the two sessions into one
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
	testHarness.processWatermark(new Watermark(11501));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
	expected.add(new Watermark(11501));

	// new session
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
	testHarness.processWatermark(new Watermark(14600));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
	expected.add(new Watermark(14600));

	// this is side output as late
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));

	// this is also side output as late (we test that they are not accidentally merged)
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10100));
	sideExpected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 10100));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
	testHarness.processWatermark(new Watermark(20000));

	expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
	expected.add(new Watermark(20000));

	testHarness.processWatermark(new Watermark(100000));

	expected.add(new Watermark(100000));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	ConcurrentLinkedQueue<StreamRecord<Tuple2<String, Integer>>> sideActual = testHarness.getSideOutput(lateOutputTag);

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	TestHarnessUtil.assertOutputEqualsSorted("SideOutput was not correct.", sideExpected, (Iterable) sideActual, new Tuple2ResultSortComparator());

	testHarness.close();
}