Java Code Examples for org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness#processElement()

The following examples show how to use org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness#processElement() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamTaskOperatorTimerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOperatorYieldExecutesSelectedTimers() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
		OneInputStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOperatorChain(new OperatorID(), new TestOperatorFactory())
		.chain(new OperatorID(), new TestOperatorFactory(), StringSerializer.INSTANCE)
		.finish();

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final String trigger = TRIGGER_PREFIX + 42;
	testHarness.processElement(new StreamRecord<>(trigger));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	List<String> events = new ArrayList<>();
	testHarness.getOutput().forEach(element -> events.add(((StreamRecord<String>) element).getValue()));
	assertThat(events, is(Arrays.asList(trigger, RESULT_PREFIX + "1:0", RESULT_PREFIX + "0:0")));
}
Example 2
Source File: StatefulOperatorChainedTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void processRecords(OneInputStreamTaskTestHarness<String, String> testHarness) throws Exception {
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>("10"), 0, 0);
	testHarness.processElement(new StreamRecord<>("20"), 0, 0);
	testHarness.processElement(new StreamRecord<>("30"), 0, 0);

	testHarness.waitForInputProcessing();

	expectedOutput.add(new StreamRecord<>("10"));
	expectedOutput.add(new StreamRecord<>("20"));
	expectedOutput.add(new StreamRecord<>("30"));
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
Example 3
Source File: StatefulOperatorChainedTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void processRecords(OneInputStreamTaskTestHarness<String, String> testHarness) throws Exception {
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>("10"), 0, 0);
	testHarness.processElement(new StreamRecord<>("20"), 0, 0);
	testHarness.processElement(new StreamRecord<>("30"), 0, 0);

	testHarness.waitForInputProcessing();

	expectedOutput.add(new StreamRecord<>("10"));
	expectedOutput.add(new StreamRecord<>("20"));
	expectedOutput.add(new StreamRecord<>("30"));
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
Example 4
Source File: StatefulOperatorChainedTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void processRecords(OneInputStreamTaskTestHarness<String, String> testHarness) throws Exception {
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>("10"), 0, 0);
	testHarness.processElement(new StreamRecord<>("20"), 0, 0);
	testHarness.processElement(new StreamRecord<>("30"), 0, 0);

	testHarness.waitForInputProcessing();

	expectedOutput.add(new StreamRecord<>("10"));
	expectedOutput.add(new StreamRecord<>("20"));
	expectedOutput.add(new StreamRecord<>("30"));
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
Example 5
Source File: MailboxOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAvoidTaskStarvation() throws Exception {
	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
		OneInputStreamTask::new,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO);

	final int maxProcessingElements = 3;

	testHarness.setupOperatorChain(new OperatorID(), new ReplicatingMailOperatorFactory(maxProcessingElements))
		.chain(new OperatorID(), new ReplicatingMailOperatorFactory(maxProcessingElements), IntSerializer.INSTANCE)
		.finish();

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	for (int i = 0; i < maxProcessingElements; i++) {
		testHarness.processElement(new StreamRecord<>(0));
	}

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// with each input two mails should be processed, one of each operator in the chain
	List<Integer> expected = new ArrayList<>();
	for (int i = 0; i < maxProcessingElements; i++) {
		expected.add(i * 2);
	}
	List<Integer> numMailsProcessed = testHarness.getOutput().stream()
		.map(element -> ((StreamRecord<Integer>) element).getValue())
		.collect(Collectors.toList());
	assertThat(numMailsProcessed, is(expected));
}
Example 6
Source File: AsyncWaitOperatorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(false);

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	final AsyncWaitOperator<Integer, Integer> headOperator =
			operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
	streamConfig.setStreamOperator(headOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}
Example 7
Source File: AsyncWaitOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(false);

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	final AsyncWaitOperator<Integer, Integer> headOperator =
			operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
	streamConfig.setStreamOperator(headOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}
Example 8
Source File: AsyncWaitOperatorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(new MyAsyncFunction(), new MyAsyncFunction());

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	streamConfig.setStreamOperatorFactory(
			operatorChainStreamConfig.getStreamOperatorFactory(AsyncWaitOperatorTest.class.getClassLoader()));

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	List<Object> expectedOutput = new LinkedList<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}