Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#endInput()

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamElement} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 1, AsyncDataStream.OutputMode.UNORDERED);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(42, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.endInput();
		harness.close();
	}

	// check that we actually outputted the result of the single input
	assertEquals(Arrays.asList(new StreamRecord(42 * 2, 1L)), new ArrayList<>(harness.getOutput()));

	// check that we have cancelled our registered timeout
	assertEquals(0, harness.getProcessingTimeService().getNumActiveTimers());
}
 

@Test
public void testTemporalInnerAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jark"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTemporalInnerAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.INNER_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTemporalLeftAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jark"));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTemporalLeftAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(insertRecord(1, "a"));
		testHarness.processElement(insertRecord(2, "b"));
		testHarness.processElement(insertRecord(3, "c"));
		testHarness.processElement(insertRecord(4, "d"));
		testHarness.processElement(insertRecord(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

/**
 * Delay a while before async invocation to check whether end input waits for all elements finished or not.
 */
@Test
public void testEndInput() throws Exception {
	final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new DelayedAsyncFunction(10),
		-1,
		2,
		AsyncDataStream.OutputMode.ORDERED);

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));

	testHarness.open();

	try {
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
			testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
			testHarness.processWatermark(new Watermark(initialTime + 2));
			testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
		}

		// wait until all async collectors in the buffer have been emitted out.
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.endInput();
		}

		TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
	} finally {
		synchronized (testHarness.getCheckpointLock()) {
			testHarness.close();
		}
	}
}
 

private void testEventTime(AsyncDataStream.OutputMode mode) throws Exception {
	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 2, mode);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
		testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
		testHarness.processWatermark(new Watermark(initialTime + 2));
		testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));

	if (AsyncDataStream.OutputMode.ORDERED == mode) {
		TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
	}
	else {
		Object[] jobOutputQueue = testHarness.getOutput().toArray();

		Assert.assertEquals("Watermark should be at index 2", new Watermark(initialTime + 2), jobOutputQueue[2]);
		Assert.assertEquals("StreamRecord 3 should be at the end", new StreamRecord<>(6, initialTime + 3), jobOutputQueue[3]);

		TestHarnessUtil.assertOutputEqualsSorted(
				"Output for StreamRecords does not match",
				expectedOutput,
				testHarness.getOutput(),
				new StreamRecordComparator());
	}
}
 

private void testProcessingTime(AsyncDataStream.OutputMode mode) throws Exception {
	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 6, mode);

	final long initialTime = 0L;
	final Queue<Object> expectedOutput = new ArrayDeque<>();

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
		testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
		testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
		testHarness.processElement(new StreamRecord<>(4, initialTime + 4));
		testHarness.processElement(new StreamRecord<>(5, initialTime + 5));
		testHarness.processElement(new StreamRecord<>(6, initialTime + 6));
		testHarness.processElement(new StreamRecord<>(7, initialTime + 7));
		testHarness.processElement(new StreamRecord<>(8, initialTime + 8));
	}

	expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
	expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
	expectedOutput.add(new StreamRecord<>(6, initialTime + 3));
	expectedOutput.add(new StreamRecord<>(8, initialTime + 4));
	expectedOutput.add(new StreamRecord<>(10, initialTime + 5));
	expectedOutput.add(new StreamRecord<>(12, initialTime + 6));
	expectedOutput.add(new StreamRecord<>(14, initialTime + 7));
	expectedOutput.add(new StreamRecord<>(16, initialTime + 8));

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	if (mode == AsyncDataStream.OutputMode.ORDERED) {
		TestHarnessUtil.assertOutputEquals("ORDERED Output was not correct.", expectedOutput, testHarness.getOutput());
	}
	else {
		TestHarnessUtil.assertOutputEqualsSorted(
				"UNORDERED Output was not correct.",
				expectedOutput,
				testHarness.getOutput(),
				new StreamRecordComparator());
	}
}
 

private void testAsyncTimeout(
		LazyAsyncFunction lazyAsyncFunction,
		Optional<Class<? extends Throwable>> expectedException,
		StreamRecord<Integer>... expectedRecords) throws Exception {
	final long timeout = 10L;

	final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness =
		createTestHarness(lazyAsyncFunction, timeout, 2, AsyncDataStream.OutputMode.ORDERED);

	final MockEnvironment mockEnvironment = testHarness.getEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	final long initialTime = 0L;
	final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.setProcessingTime(initialTime);

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(new StreamRecord<>(1, initialTime));
		testHarness.setProcessingTime(initialTime + 5L);
		testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
	}

	// trigger the timeout of the first stream record
	testHarness.setProcessingTime(initialTime + timeout + 1L);

	// allow the second async stream record to be processed
	lazyAsyncFunction.countDown();

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.endInput();
		testHarness.close();
	}

	expectedOutput.addAll(Arrays.asList(expectedRecords));

	TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());

	if (expectedException.isPresent()) {
		assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
		assertTrue(ExceptionUtils.findThrowable(
			mockEnvironment.getActualExternalFailureCause().get(),
			expectedException.get()).isPresent());
	}
}
 

/**
 * Tests that the AysncWaitOperator can restart if checkpointed queue was full.
 *
 * <p>See FLINK-7949
 */
@Test(timeout = 10000)
public void testRestartWithFullQueue() throws Exception {
	final int capacity = 10;

	// 1. create the snapshot which contains capacity + 1 elements
	final CompletableFuture<Void> trigger = new CompletableFuture<>();

	final OneInputStreamOperatorTestHarness<Integer, Integer> snapshotHarness = createTestHarness(
		new ControllableAsyncFunction<>(trigger), // the NoOpAsyncFunction is like a blocking function
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	snapshotHarness.open();

	final OperatorSubtaskState snapshot;

	final ArrayList<Integer> expectedOutput = new ArrayList<>(capacity);

	try {
		synchronized (snapshotHarness.getCheckpointLock()) {
			for (int i = 0; i < capacity; i++) {
				snapshotHarness.processElement(i, 0L);
				expectedOutput.add(i);
			}
		}

		synchronized (snapshotHarness.getCheckpointLock()) {
			// execute the snapshot within the checkpoint lock, because then it is guaranteed
			// that the lastElementWriter has written the exceeding element
			snapshot = snapshotHarness.snapshot(0L, 0L);
		}

		// trigger the computation to make the close call finish
		trigger.complete(null);
	} finally {
		synchronized (snapshotHarness.getCheckpointLock()) {
			snapshotHarness.close();
		}
	}

	// 2. restore the snapshot and check that we complete
	final OneInputStreamOperatorTestHarness<Integer, Integer> recoverHarness = createTestHarness(
		new ControllableAsyncFunction<>(CompletableFuture.completedFuture(null)),
		1000L,
		capacity,
		AsyncDataStream.OutputMode.ORDERED);

	recoverHarness.initializeState(snapshot);

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.open();
	}

	synchronized (recoverHarness.getCheckpointLock()) {
		recoverHarness.endInput();
		recoverHarness.close();
	}

	final ConcurrentLinkedQueue<Object> output = recoverHarness.getOutput();

	final List<Integer> outputElements = output.stream()
			.map(r -> ((StreamRecord<Integer>) r).getValue())
			.collect(Collectors.toList());

	assertThat(outputElements, Matchers.equalTo(expectedOutput));
}