Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#endInput()

The following examples show how to use org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#endInput() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamElement} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 1, AsyncDataStream.OutputMode.UNORDERED);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(42, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.endInput();
		harness.close();
	}

	// check that we actually outputted the result of the single input
	assertEquals(Arrays.asList(new StreamRecord(42 * 2, 1L)), new ArrayList<>(harness.getOutput()));

	// check that we have cancelled our registered timeout
	assertEquals(0, harness.getProcessingTimeService().getNumActiveTimers());
}
 
Example 2
@Test
public void testTemporalInnerAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jark"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 3
@Test
public void testTemporalInnerAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 4
@Test
public void testTemporalLeftAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jark"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 5
@Test
public void testTemporalLeftAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 
Example 6
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Delay a while before async invocation to check whether end input waits for all elements finished or not.
 */
@Test
public void testEndInput() throws Exception {
	final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new DelayedAsyncFunction(10),
		-1,
		2,
		AsyncDataStream.OutputMode.ORDERED);

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));

	testHarness.open();

	try {
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
			testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
			testHarness.processWatermark(new Watermark(initialTime + 2));
			testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
		}

		// wait until all async collectors in the buffer have been emitted out.
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.endInput();
		}

		TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
	} finally {
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.close();
		}
	}
}
 
Example 7
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testEventTime(AsyncDataStream.OutputMode mode) throws Exception {
	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 2, mode);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
		testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
		testHarness.processWatermark(new Watermark(initialTime + 2));
		testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));

	if (AsyncDataStream.OutputMode.ORDERED == mode) {
		TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
	}
	else {
		Object[] jobOutputQueue = testHarness.getOutput().toArray();

		Assert.assertEquals("Watermark should be at index 2", new Watermark(initialTime + 2), jobOutputQueue[2]);
		Assert.assertEquals("StreamRecord 3 should be at the end", new StreamRecord<>(6, initialTime + 3), jobOutputQueue[3]);

		TestHarnessUtil.assertOutputEqualsSorted(
				"Output for StreamRecords does not match",
				expectedOutput,
				testHarness.getOutput(),
				new StreamRecordComparator());
	}
}
 
Example 8
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testProcessingTime(AsyncDataStream.OutputMode mode) throws Exception {
	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 6, mode);

	final long initialTime = 0L;
	final Queue<Object> expectedOutput = new ArrayDeque<>();

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
		testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
		testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
		testHarness.processElement(new StreamRecord<>(4, initialTime + 4));
		testHarness.processElement(new StreamRecord<>(5, initialTime + 5));
		testHarness.processElement(new StreamRecord<>(6, initialTime + 6));
		testHarness.processElement(new StreamRecord<>(7, initialTime + 7));
		testHarness.processElement(new StreamRecord<>(8, initialTime + 8));
	}

	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(8, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(10, initialTime + 5));
	expectedOutput.add(new StreamRecord<>(12, initialTime + 6));
	expectedOutput.add(new StreamRecord<>(14, initialTime + 7));
	expectedOutput.add(new StreamRecord<>(16, initialTime + 8));

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	if (mode == AsyncDataStream.OutputMode.ORDERED) {
		TestHarnessUtil.assertOutputEquals("ORDERED Output was not correct.", expectedOutput, testHarness.getOutput());
	}
	else {
		TestHarnessUtil.assertOutputEqualsSorted(
				"UNORDERED Output was not correct.",
				expectedOutput,
				testHarness.getOutput(),
				new StreamRecordComparator());
	}
}
 
Example 9
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testAsyncTimeout(
		LazyAsyncFunction lazyAsyncFunction,
		Optional<Class<? extends Throwable>> expectedException,
		StreamRecord<Integer>... expectedRecords) throws Exception {
	final long timeout = 10L;

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(lazyAsyncFunction, timeout, 2, AsyncDataStream.OutputMode.ORDERED);

	final MockEnvironment mockEnvironment = testHarness.getEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(initialTime);

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime));
		testHarness.setProcessingTime(initialTime + 5L);
		testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
	}

	// trigger the timeout of the first stream record
	testHarness.setProcessingTime(initialTime + timeout + 1L);

	// allow the second async stream record to be processed
	lazyAsyncFunction.countDown();

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	expectedOutput.addAll(Arrays.asList(expectedRecords));

	TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());

	if (expectedException.isPresent()) {
		assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
		assertTrue(ExceptionUtils.findThrowable(
			mockEnvironment.getActualExternalFailureCause().get(),
			expectedException.get()).isPresent());
	}
}
 
Example 10
Source Project: flink   File: AsyncWaitOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the AysncWaitOperator can restart if checkpointed queue was full.
 *
 * <p>See FLINK-7949
 */
@Test(timeout = 10000)
public void testRestartWithFullQueue() throws Exception {
	final int capacity = 10;

	// 1. create the snapshot which contains capacity + 1 elements
	final CompletableFuture<Void> trigger = new CompletableFuture<>();

	final OneInputStreamOperatorTestHarness<Integer, Integer> snapshotHarness = createTestHarness(
		new ControllableAsyncFunction<>(trigger), // the NoOpAsyncFunction is like a blocking function
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	snapshotHarness.open();

	final OperatorSubtaskState snapshot;

	final ArrayList<Integer> expectedOutput = new ArrayList<>(capacity);

	try {
		synchronized (snapshotHarness.getCheckpointLock()) {
			for (int i = 0; i < capacity; i++) {
				snapshotHarness.processElement(i, 0L);
				expectedOutput.add(i);
			}
		}

		synchronized (snapshotHarness.getCheckpointLock()) {
			// execute the snapshot within the checkpoint lock, because then it is guaranteed
			// that the lastElementWriter has written the exceeding element
			snapshot = snapshotHarness.snapshot(0L, 0L);
		}

		// trigger the computation to make the close call finish
		trigger.complete(null);
	} finally {
		synchronized (snapshotHarness.getCheckpointLock()) {
			snapshotHarness.close();
		}
	}

	// 2. restore the snapshot and check that we complete
	final OneInputStreamOperatorTestHarness<Integer, Integer> recoverHarness = createTestHarness(
		new ControllableAsyncFunction<>(CompletableFuture.completedFuture(null)),
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	recoverHarness.initializeState(snapshot);

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.open();
	}

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.endInput();
		recoverHarness.close();
	}

	final ConcurrentLinkedQueue<Object> output = recoverHarness.getOutput();

	final List<Integer> outputElements = output.stream()
			.map(r -> ((StreamRecord<Integer>) r).getValue())
			.collect(Collectors.toList());

	assertThat(outputElements, Matchers.equalTo(expectedOutput));
}