org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness Java Examples

@Test
public void testRunOutOfProducersInThePool() throws Exception {
	String topic = "flink-kafka-run-out-of-producers";

	try (OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic)) {

		testHarness.setup();
		testHarness.open();

		for (int i = 0; i < FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE * 2; i++) {
			testHarness.processElement(i, i * 2);
			testHarness.snapshot(i, i * 2 + 1);
		}
	}
	catch (Exception ex) {
		if (!ex.getCause().getMessage().startsWith("Too many ongoing")) {
			throw ex;
		}
	}
	deleteTestTopic(topic);
	checkProducerLeak();
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new NoOpAsyncFunction<>(), 10L, 2, outputMode);

	harness.getEnvironment().setExpectedExternalFailureCause(Throwable.class);
	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

private OneInputStreamOperatorTestHarness<String, Object> createTestSink(File dataDir, int totalParallelism, int taskIdx) throws Exception {
	BucketingSink<String> sink = new BucketingSink<String>(dataDir.getAbsolutePath())
		.setBucketer(new Bucketer<String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public Path getBucketPath(Clock clock, Path basePath, String element) {
				return new Path(basePath, element);
			}
		})
		.setWriter(new StringWriter<String>())
		.setPartPrefix(PART_PREFIX)
		.setPendingPrefix("")
		.setInactiveBucketCheckInterval(5 * 60 * 1000L)
		.setInactiveBucketThreshold(5 * 60 * 1000L)
		.setPendingSuffix(PENDING_SUFFIX)
		.setInProgressSuffix(IN_PROGRESS_SUFFIX);

	return createTestSink(sink, totalParallelism, taskIdx);
}
 

private OneInputStreamOperatorTestHarness<IN, OUT> getTestHarness(Configuration config) throws Exception {
	RowType dataType = new RowType(Arrays.asList(
		new RowType.RowField("f1", new VarCharType()),
		new RowType.RowField("f2", new VarCharType()),
		new RowType.RowField("f3", new BigIntType())));
	AbstractPythonScalarFunctionOperator<IN, OUT, UDFIN> operator = getTestOperator(
		config,
		new PythonFunctionInfo[] {
			new PythonFunctionInfo(
				AbstractPythonScalarFunctionRunnerTest.DummyPythonFunction.INSTANCE,
				new Integer[]{0})
		},
		dataType,
		dataType,
		new int[]{2},
		new int[]{0, 1}
	);

	OneInputStreamOperatorTestHarness<IN, OUT> testHarness =
		new OneInputStreamOperatorTestHarness<>(operator);
	testHarness.getStreamConfig().setManagedMemoryFraction(0.5);
	testHarness.setup(getOutputTypeSerializer(dataType));
	return testHarness;
}
 

@Test
public void testAssignerWithMultipleWindowsForAggregate() throws Exception {
	WindowAssigner<TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<TimeWindow> mockTrigger = mockTrigger();
	NamespaceAggsHandleFunction<TimeWindow> mockAggregate = mockAggsHandleFunction();

	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, mockAggregate, 0L);

	testHarness.open();

	when(mockAssigner.assignWindows(any(), anyLong()))
			.thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));

	shouldFireOnElement(mockTrigger);

	testHarness.processElement(insertRecord("String", 1, 0L));

	verify(mockAggregate, times(2)).getValue(anyTimeWindow());
	verify(mockAggregate, times(1)).getValue(eq(new TimeWindow(0, 2)));
	verify(mockAggregate, times(1)).getValue(eq(new TimeWindow(2, 4)));
}
 

@Test
public void testRecoverCommittedTransaction() throws Exception {
	String topic = "flink-kafka-producer-recover-committed-transaction";

	OneInputStreamOperatorTestHarness<Integer, Object> testHarness = createTestHarness(topic);

	testHarness.setup();
	testHarness.open(); // producerA - start transaction (txn) 0
	testHarness.processElement(42, 0); // producerA - write 42 in txn 0
	OperatorSubtaskState checkpoint0 = testHarness.snapshot(0, 1); // producerA - pre commit txn 0, producerB - start txn 1
	testHarness.processElement(43, 2); // producerB - write 43 in txn 1
	testHarness.notifyOfCompletedCheckpoint(0); // producerA - commit txn 0 and return to the pool
	testHarness.snapshot(1, 3); // producerB - pre txn 1,  producerA - start txn 2
	testHarness.processElement(44, 4); // producerA - write 44 in txn 2
	testHarness.close(); // producerA - abort txn 2

	testHarness = createTestHarness(topic);
	testHarness.initializeState(checkpoint0); // recover state 0 - producerA recover and commit txn 0
	testHarness.close();

	assertExactlyOnceForTopic(createProperties(), topic, 0, Arrays.asList(42));

	deleteTestTopic(topic);
	checkProducerLeak();
}
 

@Test
public void punctuatedWatermarksDoNotRegress() throws Exception {
	OneInputStreamOperatorTestHarness<Tuple2<Boolean, Long>, Tuple2<Boolean, Long>> testHarness = createTestHarness(
			WatermarkStrategy
					.forGenerator((ctx) -> new PunctuatedWatermarkGenerator())
					.withTimestampAssigner((ctx) -> new TupleExtractor()));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(true, 4L), 1));

	assertThat(pollNextStreamRecord(testHarness), streamRecord(new Tuple2<>(true, 4L), 4L));
	assertThat(pollNextLegacyWatermark(testHarness), is(legacyWatermark(4L)));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(true, 2L), 1));

	assertThat(pollNextStreamRecord(testHarness), streamRecord(new Tuple2<>(true, 2L), 2L));
	assertThat(testHarness.getOutput(), empty());
}
 

@Test
public void testAssignerWithMultipleWindowsForAggregate() throws Exception {
	WindowAssigner<TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<TimeWindow> mockTrigger = mockTrigger();
	NamespaceAggsHandleFunction<TimeWindow> mockAggregate = mockAggsHandleFunction();

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, mockAggregate, 0L);

	testHarness.open();

	when(mockAssigner.assignWindows(any(), anyLong()))
			.thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));

	shouldFireOnElement(mockTrigger);

	testHarness.processElement(record("String", 1, 0L));

	verify(mockAggregate, times(2)).getValue(anyTimeWindow());
	verify(mockAggregate, times(1)).getValue(eq(new TimeWindow(0, 2)));
	verify(mockAggregate, times(1)).getValue(eq(new TimeWindow(2, 4)));
}
 

/**
 * This tests checks whether there is some resource leak in form of growing threads number.
 */
public void resourceCleanUp(Semantic semantic) throws Exception {
	String topic = "flink-kafka-producer-resource-cleanup-" + semantic;

	final int allowedEpsilonThreadCountGrow = 50;

	Optional<Integer> initialActiveThreads = Optional.empty();
	for (int i = 0; i < allowedEpsilonThreadCountGrow * 2; i++) {
		try (OneInputStreamOperatorTestHarness<Integer, Object> testHarness1 =
				createTestHarness(topic, 1, 1, 0, semantic)) {
			testHarness1.setup();
			testHarness1.open();
		}

		if (initialActiveThreads.isPresent()) {
			assertThat("active threads count",
				Thread.activeCount(),
				lessThan(initialActiveThreads.get() + allowedEpsilonThreadCountGrow));
		}
		else {
			initialActiveThreads = Optional.of(Thread.activeCount());
		}
		checkProducerLeak();
	}
}
 

@Test
public void testNullOutputTagRefusal() throws Exception {
	KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(
			operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(17);
	try {
		expectedException.expect(IllegalArgumentException.class);
		testHarness.processElement(new StreamRecord<>(5));
	} finally {
		testHarness.close();
	}
}
 

static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomRescalingTestSink(
		final File outDir,
		final int totalParallelism,
		final int taskIdx,
		final long bucketCheckInterval,
		final BucketAssigner<Tuple2<String, Integer>, String> bucketer,
		final Encoder<Tuple2<String, Integer>> writer,
		final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy,
		final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception {

	StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink
			.forRowFormat(new Path(outDir.toURI()), writer)
			.withBucketAssigner(bucketer)
			.withRollingPolicy(rollingPolicy)
			.withBucketCheckInterval(bucketCheckInterval)
			.withBucketFactory(bucketFactory)
			.build();

	return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx);
}
 

/**
 * Ensures Jackson cache is cleaned to get rid of any references to the Flink Classloader. See
 * https://jira.apache.org/jira/browse/BEAM-6460
 */
@Test
public void testRemoveCachedClassReferences() throws Exception {
  OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness =
      new OneInputStreamOperatorTestHarness<>(getOperatorForCleanupInspection());

  LRUMap typeCache =
      (LRUMap) Whitebox.getInternalState(TypeFactory.defaultInstance(), "_typeCache");
  assertThat(typeCache.size(), greaterThan(0));
  testHarness.open();
  testHarness.close();
  assertThat(typeCache.size(), is(0));
}
 

/**
 * Test ensuring that if a snapshot call happens right after an async exception is caught, it should be rethrown.
 */
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test
public void testAsyncErrorRethrownOnCheckpoint() throws Throwable {
	final DummyFlinkKinesisProducer<String> producer = new DummyFlinkKinesisProducer<>(new SimpleStringSchema());

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));

	testHarness.open();

	testHarness.processElement(new StreamRecord<>("msg-1"));

	producer.getPendingRecordFutures().get(0).setException(new Exception("artificial async exception"));

	try {
		testHarness.snapshot(123L, 123L);
	} catch (Exception e) {
		// the next checkpoint should rethrow the async exception
		Assert.assertTrue(ExceptionUtils.findThrowableWithMessage(e, "artificial async exception").isPresent());

		// test succeeded
		return;
	}

	Assert.fail();
}
 

private OneInputStreamOperatorTestHarness<BaseRow, BaseRow> createTestHarness(
		MiniBatchDeduplicateKeepFirstRowFunction func)
		throws Exception {
	CountBundleTrigger<Tuple2<String, String>> trigger = new CountBundleTrigger<>(3);
	KeyedMapBundleOperator op = new KeyedMapBundleOperator(func, trigger);
	return new KeyedOneInputStreamOperatorTestHarness<>(op, rowKeySelector, rowKeySelector.getProducedType());
}
 

private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness(
	BlockingFileInputFormat format,
	int noOfTasks,
	int taskIdx) throws Exception {
	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(
			new ContinuousFileReaderOperatorFactory<>(format, TypeExtractor.getInputFormatTypes(format), new ExecutionConfig()),
			maxParallelism,
			noOfTasks,
			taskIdx);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
	return testHarness;
}
 

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeSessionWindowsWithCountTriggerInMintConditionSnapshot() throws Exception {

	final int sessionSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector<String>(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new SessionWindowFunction()),
			PurgingTrigger.of(CountTrigger.of(4)),
			0,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	// do snapshot and save to file
	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
	OperatorSnapshotUtil.writeStateHandle(
		snapshot,
		"src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink" + flinkGenerateSavepointVersion + "-snapshot");

	testHarness.close();
}
 

public static <T> OneInputStreamOperatorTestHarness<Event, T> getCepTestHarness(
	CepOperator<Event, Integer, T> cepOperator) throws Exception {
	KeySelector<Event, Integer> keySelector = new TestKeySelector();

	return new KeyedOneInputStreamOperatorTestHarness<>(
		cepOperator,
		keySelector,
		BasicTypeInfo.INT_TYPE_INFO);
}
 

@Test
public void testTemporalLeftAsyncJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(record(1, "a"));
		testHarness.processElement(record(2, "b"));
		testHarness.processElement(record(3, "c"));
		testHarness.processElement(record(4, "d"));
		testHarness.processElement(record(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(1, "a", 1, "Julian"));
	expectedOutput.add(record(2, "b", null, null));
	expectedOutput.add(record(3, "c", 3, "Jackson"));
	expectedOutput.add(record(4, "d", 4, "Fabian"));
	expectedOutput.add(record(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test(timeout = DEFAULT_TEST_TIMEOUT)
public void testWaitForPendingUpdatesOnSnapshot() throws Exception {
	final TestCassandraSink casSinkFunc = new TestCassandraSink();

	try (OneInputStreamOperatorTestHarness<String, Object> testHarness = createOpenedTestHarness(casSinkFunc)) {
		CompletableFuture<ResultSet> completableFuture = new CompletableFuture<>();
		casSinkFunc.enqueueCompletableFuture(completableFuture);

		casSinkFunc.invoke("hello");
		Assert.assertEquals(1, casSinkFunc.getAcquiredPermits());

		final CountDownLatch latch = new CountDownLatch(1);
		Thread t = new CheckedThread("Flink-CassandraSinkBaseTest") {
			@Override
			public void go() throws Exception {
				testHarness.snapshot(123L, 123L);
				latch.countDown();
			}
		};
		t.start();
		while (t.getState() != Thread.State.WAITING) {
			Thread.sleep(5);
		}

		Assert.assertEquals(1, casSinkFunc.getAcquiredPermits());
		completableFuture.complete(null);
		latch.await();
		Assert.assertEquals(0, casSinkFunc.getAcquiredPermits());
	}
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * Extracts the result values form the test harness and clear the output queue.
 */
@SuppressWarnings({"unchecked", "rawtypes"})
private <T> List<T> extractResult(OneInputStreamOperatorTestHarness<?, T> testHarness) {
	List<StreamRecord<? extends T>> streamRecords = testHarness.extractOutputStreamRecords();
	List<T> result = new ArrayList<>();
	for (Object in : streamRecords) {
		if (in instanceof StreamRecord) {
			result.add((T) ((StreamRecord) in).getValue());
		}
	}
	testHarness.getOutput().clear();
	return result;
}
 

/** Tests that any item failure in the listener callbacks is rethrown on an immediately following invoke call. */
@Test
public void testItemFailureRethrownOnInvoke() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and its mock item failures
	sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record")));
	testHarness.processElement(new StreamRecord<>("msg"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class));

	// manually execute the next bulk request
	sink.manualBulkRequestWithAllPendingRequests();

	try {
		testHarness.processElement(new StreamRecord<>("next msg"));
	} catch (Exception e) {
		// the invoke should have failed with the failure
		Assert.assertTrue(e.getCause().getMessage().contains("artificial failure for record"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 

@Test
public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception {
	final int gapSize = 3;
	final long lateness = 10;

	ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc =
		new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator =
		new WindowOperator<>(
			EventTimeSessionWindows.withGap(Time.seconds(gapSize)),
			new TimeWindow.Serializer(),
			new TupleKeySelector(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			windowStateDesc,
			new InternalIterableWindowFunction<>(new PassThroughFunction()),
			EventTimeTrigger.create(),
			lateness,
			null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		createTestHarness(operator);

	testHarness.open();

	ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
	testHarness.processWatermark(new Watermark(4998));

	expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
	expected.add(new Watermark(4998));

	testHarness.processWatermark(new Watermark(14600));
	expected.add(new Watermark(14600));

	ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
	testHarness.close();
}
 

@Test
public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception {

	OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {};

	try (
		OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness(
			createCepOperator(
				extractCurrentProcessingTimeAndNames(2, sideOutputTag),
				new NFATimingOutFactory(),
				EVENT_TIME))) {
		harness.open();

		// events out of order to test if internal sorting does not mess up the timestamps
		harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord());
		harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord());
		harness.processElement(event().withName("C").withTimestamp(3).asStreamRecord());

		harness.setProcessingTime(100);
		harness.processWatermark(22);

		assertOutput(harness.getOutput())
			.nextElementEquals("100:C:A")
			.watermarkEquals(22)
			.hasNoMoreElements();

		assertOutput(harness.getSideOutput(sideOutputTag))
			.nextElementEquals("100:A")
			.hasNoMoreElements();
	}
}
 

@Test
public void testDisableGenerateRetractionAndOutputRankNumber() throws Exception {
	AbstractTopNFunction func = createFunction(RankType.ROW_NUMBER, new ConstantRankRange(1, 2), false,
			true);
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createTestHarness(func);
	testHarness.open();
	testHarness.processElement(record("book", 1L, 12));
	testHarness.processElement(record("book", 2L, 19));
	testHarness.processElement(record("book", 4L, 11));
	testHarness.processElement(record("book", 5L, 11));
	testHarness.processElement(record("fruit", 4L, 33));
	testHarness.processElement(record("fruit", 3L, 44));
	testHarness.processElement(record("fruit", 5L, 22));
	testHarness.close();

	// Notes: Retract message will not be sent if disable generate retraction and output rankNumber.
	// Because partition key + rankNumber decomposes a uniqueKey.
	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record("book", 1L, 12, 1L));
	expectedOutput.add(record("book", 2L, 19, 2L));
	expectedOutput.add(record("book", 4L, 11, 1L));
	expectedOutput.add(record("book", 1L, 12, 2L));
	expectedOutput.add(record("book", 5L, 11, 2L));
	expectedOutput.add(record("fruit", 4L, 33, 1L));
	expectedOutput.add(record("fruit", 3L, 44, 2L));
	expectedOutput.add(record("fruit", 5L, 22, 1L));
	expectedOutput.add(record("fruit", 4L, 33, 2L));
	assertorWithRowNumber.assertOutputEqualsSorted("output wrong.", expectedOutput, testHarness.getOutput());
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void periodicWatermarksOnlyEmitOnPeriodicEmit() throws Exception {
	OneInputStreamOperatorTestHarness<Long, Long> testHarness = createTestHarness(
			WatermarkStrategy
					.forGenerator((ctx) -> new PeriodicWatermarkGenerator())
					.withTimestampAssigner((ctx) -> new LongExtractor()));

	testHarness.processElement(new StreamRecord<>(2L, 1));

	assertThat(pollNextStreamRecord(testHarness), streamRecord(2L, 2L));
	assertThat(testHarness.getOutput(), empty());
}
 

@Test
public void testTemporalLeftAsyncJoin() throws Exception {
	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITHOUT_FILTER);

	testHarness.open();

	synchronized (testHarness.getCheckpointLock()) {
		testHarness.processElement(record(1, "a"));
		testHarness.processElement(record(2, "b"));
		testHarness.processElement(record(3, "c"));
		testHarness.processElement(record(4, "d"));
		testHarness.processElement(record(5, "e"));
	}

	// wait until all async collectors in the buffer have been emitted out.
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(1, "a", 1, "Julian"));
	expectedOutput.add(record(2, "b", null, null));
	expectedOutput.add(record(3, "c", 3, "Jark"));
	expectedOutput.add(record(3, "c", 3, "Jackson"));
	expectedOutput.add(record(4, "d", 4, "Fabian"));
	expectedOutput.add(record(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testProject() throws Exception {

	TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor
			.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));

	int[] fields = new int[]{4, 4, 3};

	TupleSerializer<Tuple3<Integer, Integer, String>> serializer =
			new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType))
					.createSerializer(new ExecutionConfig());
	@SuppressWarnings("unchecked")
	StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator =
			new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(
					fields, serializer);

	OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));

	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testIdealCircumstances() throws Exception {
	S sink = createSink();

	OneInputStreamOperatorTestHarness<IN, IN> testHarness =
			new OneInputStreamOperatorTestHarness<>(sink);

	testHarness.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	for (int x = 0; x < 20; x++) {
		testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
		elementCounter++;
	}

	testHarness.snapshot(snapshotCount++, 0);
	testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);

	verifyResultsIdealCircumstances(sink);
}