org.apache.flink.runtime.state.TestTaskStateManager Java Examples

public StreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
 

@Test
public void testNotifyCheckpointComplete() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.build();

	final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);

	long checkpointId = 42L;
	{
		subtaskCheckpointCoordinator.notifyCheckpointComplete(checkpointId, operatorChain, () -> true);
		assertEquals(checkpointId, stateManager.getNotifiedCompletedCheckpointId());
	}

	long newCheckpointId = checkpointId + 1;
	{
		subtaskCheckpointCoordinator.notifyCheckpointComplete(newCheckpointId, operatorChain, () -> false);
		// even task is not running, state manager could still receive the notification.
		assertEquals(newCheckpointId, stateManager.getNotifiedCompletedCheckpointId());
	}
}
 

@Test
public void testNotifyCheckpointAbortedAfterAsyncPhase() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.build();

	final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);

	long checkpointId = 42L;
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	assertEquals(checkpointId, stateManager.getNotifiedAbortedCheckpointId());
}
 

public StreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<OUT>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
 

public StreamTaskTestHarness(
	FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
	TypeInformation<OUT> outputType,
	LocalRecoveryConfig localRecoveryConfig) {
	this.taskFactory = checkNotNull(taskFactory);
	this.memorySize = DEFAULT_MEMORY_MANAGER_SIZE;
	this.bufferSize = DEFAULT_NETWORK_BUFFER_SIZE;

	this.jobConfig = new Configuration();
	this.taskConfig = new Configuration();
	this.executionConfig = new ExecutionConfig();

	streamConfig = new StreamConfig(taskConfig);
	streamConfig.setBufferTimeout(0);

	outputSerializer = outputType.createSerializer(executionConfig);
	outputStreamRecordSerializer = new StreamElementSerializer<>(outputSerializer);

	this.taskStateManager = new TestTaskStateManager(localRecoveryConfig);
}
 

static Environment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	Environment env = mock(Environment.class);
	when(env.getJobID()).thenReturn(new JobID());
	when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
	when(env.getIOManager()).thenReturn(ioMan);
	when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

	TaskInfo taskInfo = mock(TaskInfo.class);
	when(env.getTaskInfo()).thenReturn(taskInfo);
	when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);

	TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
	when(env.getTaskManagerInfo()).thenReturn(tmInfo);

	TestTaskStateManager taskStateManager = new TestTaskStateManager();
	when(env.getTaskStateManager()).thenReturn(taskStateManager);

	return env;
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig,
	TaskManagerActions taskManagerActions) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), taskManagerActions);
}
 

static Environment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	Environment env = mock(Environment.class);
	when(env.getJobID()).thenReturn(new JobID());
	when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
	when(env.getIOManager()).thenReturn(ioMan);
	when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

	TaskInfo taskInfo = mock(TaskInfo.class);
	when(env.getTaskInfo()).thenReturn(taskInfo);
	when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);

	TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
	when(env.getTaskManagerInfo()).thenReturn(tmInfo);

	TestTaskStateManager taskStateManager = new TestTaskStateManager();
	when(env.getTaskStateManager()).thenReturn(taskStateManager);

	return env;
}
 

@Test
public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setUnalignedCheckpointEnabled(true)
		.build();

	CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures());

	final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(checkpointOperator);

	long checkpointId = 42L;
	// notify checkpoint aborted before execution.
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize());

	subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	assertFalse(checkpointOperator.isCheckpointed());
	assertEquals(-1, stateManager.getReportedCheckpointId());
	assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig,
	TaskManagerActions taskManagerActions) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), taskManagerActions);
}
 

StreamTaskMailboxTestHarness(
		StreamTask<OUT, ?> streamTask,
		Queue<Object> outputList,
		StreamTestSingleInputGate[] inputGates,
		StreamMockEnvironment streamMockEnvironment) {
	this.streamTask = checkNotNull(streamTask);
	this.taskStateManager = (TestTaskStateManager) streamMockEnvironment.getTaskStateManager();
	this.inputGates = checkNotNull(inputGates);
	this.outputList = checkNotNull(outputList);
	this.streamMockEnvironment = checkNotNull(streamMockEnvironment);
	this.inputGateEnded = new boolean[inputGates.length];
}
 

public StreamTaskMailboxTestHarness<OUT> build() throws Exception {
	streamConfig.setBufferTimeout(bufferTimeout);

	TestTaskStateManager taskStateManager = new TestTaskStateManager(localRecoveryConfig);
	if (taskStateSnapshots != null) {
		taskStateManager.setReportedCheckpointId(taskStateSnapshots.keySet().iterator().next());
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(taskStateSnapshots);
	}

	StreamMockEnvironment streamMockEnvironment = new StreamMockEnvironment(
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		new MockInputSplitProvider(),
		bufferSize,
		taskStateManager);

	streamMockEnvironment.setCheckpointResponder(taskStateManager.getCheckpointResponder());
	initializeInputs(streamMockEnvironment);

	checkState(inputGates != null, "InputGates hasn't been initialised");

	StreamElementSerializer<OUT> outputStreamRecordSerializer = new StreamElementSerializer<>(outputSerializer);

	Queue<Object> outputList = new ArrayDeque<>();
	streamMockEnvironment.addOutput(outputList, outputStreamRecordSerializer);
	streamMockEnvironment.setTaskMetricGroup(taskMetricGroup);

	StreamTask<OUT, ?> task = taskFactory.apply(streamMockEnvironment);
	task.beforeInvoke();

	return new StreamTaskMailboxTestHarness<>(
		task,
		outputList,
		inputGates,
		streamMockEnvironment);
}
 

public DummyEnvironment(String taskName, int numSubTasks, int subTaskIndex, int maxParallelism) {
	this.taskInfo = new TaskInfo(taskName, maxParallelism, subTaskIndex, numSubTasks, 0);
	this.taskStateManager = new TestTaskStateManager();
	this.aggregateManager = new TestGlobalAggregateManager();
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), mock(TaskManagerActions.class));
}
 

private AbstractStreamOperatorTestHarness(
		StreamOperator<OUT> operator,
		StreamOperatorFactory<OUT> factory,
		MockEnvironment env,
		boolean environmentIsInternal,
		OperatorID operatorID) throws Exception {
	this.operator = operator;
	this.factory = factory;
	this.outputList = new ConcurrentLinkedQueue<>();
	this.sideOutputLists = new HashMap<>();

	Configuration underlyingConfig = env.getTaskConfiguration();
	this.config = new StreamConfig(underlyingConfig);
	this.config.setCheckpointingEnabled(true);
	this.config.setOperatorID(operatorID);
	this.executionConfig = env.getExecutionConfig();
	this.checkpointLock = new Object();

	this.environment = Preconditions.checkNotNull(env);

	this.taskStateManager = (TestTaskStateManager) env.getTaskStateManager();
	this.internalEnvironment = environmentIsInternal ? Optional.of(environment) : Optional.empty();

	processingTimeService = new TestProcessingTimeService();
	processingTimeService.setCurrentTime(0);

	ttlTimeProvider = new MockTtlTimeProvider();
	ttlTimeProvider.setCurrentTimestamp(0);

	this.streamTaskStateInitializer = createStreamTaskStateManager(environment, stateBackend, ttlTimeProvider);

	BiConsumer<String, Throwable> handleAsyncException = (message, t) -> {
		wasFailedExternally = true;
	};

	this.taskMailbox = new TaskMailboxImpl();

	mockTask = new MockStreamTaskBuilder(env)
		.setCheckpointLock(checkpointLock)
		.setConfig(config)
		.setExecutionConfig(executionConfig)
		.setStreamTaskStateInitializer(streamTaskStateInitializer)
		.setCheckpointStorage(checkpointStorage)
		.setTimerService(processingTimeService)
		.setHandleAsyncException(handleAsyncException)
		.setTaskMailbox(taskMailbox)
		.build();
}
 

public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

@Test
public void testOperatorMetricReuse() throws Exception {
	final TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(
		isInputSelectable ? TwoInputSelectableStreamTask::new : TwoInputStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOperatorChain(new OperatorID(), new DuplicatingOperator())
		.chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	final TaskMetricGroup taskMetricGroup = new UnregisteredMetricGroups.UnregisteredTaskMetricGroup() {
		@Override
		public OperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) {
			return new OperatorMetricGroup(NoOpMetricRegistry.INSTANCE, this, operatorID, name);
		}
	};

	final StreamMockEnvironment env = new StreamMockEnvironment(
		testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {
		@Override
		public TaskMetricGroup getMetricGroup() {
			return taskMetricGroup;
		}
	};

	final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
	final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();

	testHarness.invoke(env);
	testHarness.waitForTaskRunning();

	final int numRecords1 = 5;
	final int numRecords2 = 3;

	for (int x = 0; x < numRecords1; x++) {
		testHarness.processElement(new StreamRecord<>("hello"), 0, 0);
	}

	for (int x = 0; x < numRecords2; x++) {
		testHarness.processElement(new StreamRecord<>("hello"), 1, 0);
	}
	testHarness.waitForInputProcessing();

	assertEquals(numRecords1 + numRecords2, numRecordsInCounter.getCount());
	assertEquals((numRecords1 + numRecords2) * 2 * 2 * 2, numRecordsOutCounter.getCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

private AbstractStreamOperatorTestHarness(
		StreamOperator<OUT> operator,
		MockEnvironment env,
		boolean environmentIsInternal,
		OperatorID operatorID) throws Exception {
	this.operator = operator;
	this.outputList = new ConcurrentLinkedQueue<>();
	this.sideOutputLists = new HashMap<>();

	Configuration underlyingConfig = env.getTaskConfiguration();
	this.config = new StreamConfig(underlyingConfig);
	this.config.setCheckpointingEnabled(true);
	this.config.setOperatorID(operatorID);
	this.executionConfig = env.getExecutionConfig();
	this.closableRegistry = new CloseableRegistry();
	this.checkpointLock = new Object();

	this.environment = Preconditions.checkNotNull(env);

	this.taskStateManager = (TestTaskStateManager) env.getTaskStateManager();
	this.internalEnvironment = environmentIsInternal ? Optional.of(environment) : Optional.empty();

	processingTimeService = new TestProcessingTimeService();
	processingTimeService.setCurrentTime(0);

	this.streamTaskStateInitializer = createStreamTaskStateManager(environment, stateBackend, processingTimeService);

	BiConsumer<String, Throwable> handleAsyncException = (message, t) -> {
		wasFailedExternally = true;
	};

	mockTask = new MockStreamTaskBuilder(env)
		.setCheckpointLock(checkpointLock)
		.setConfig(config)
		.setExecutionConfig(executionConfig)
		.setStreamTaskStateInitializer(streamTaskStateInitializer)
		.setClosableRegistry(closableRegistry)
		.setCheckpointStorage(checkpointStorage)
		.setProcessingTimeService(processingTimeService)
		.setHandleAsyncException(handleAsyncException)
		.build();
}
 

private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}
 

public Task build() throws Exception {
	final JobVertexID jobVertexId = new JobVertexID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig);

	final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		allocationID,
		0,
		0,
		resultPartitions,
		inputGates,
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build(),
		mock(IOManager.class),
		shuffleEnvironment,
		kvStateService,
		new BroadcastVariableManager(),
		new TaskEventDispatcher(),
		externalResourceInfoProvider,
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	final TestingClassLoaderLease classLoaderHandle = TestingClassLoaderLease.newBuilder()
		.setGetOrResolveClassLoaderFunction((permanentBlobKeys, urls) -> new TestUserCodeClassLoader())
		.build();

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		TestingClassLoaderLease.newBuilder().build(),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

@Test
public void testOperatorMetricReuse() throws Exception {
	final TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(
		TwoInputStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOperatorChain(new OperatorID(), new DuplicatingOperator())
		.chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	final TaskMetricGroup taskMetricGroup = new UnregisteredMetricGroups.UnregisteredTaskMetricGroup() {
		@Override
		public OperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) {
			return new OperatorMetricGroup(NoOpMetricRegistry.INSTANCE, this, operatorID, name);
		}
	};

	final StreamMockEnvironment env = new StreamMockEnvironment(
		testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {
		@Override
		public TaskMetricGroup getMetricGroup() {
			return taskMetricGroup;
		}
	};

	final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
	final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();

	testHarness.invoke(env);
	testHarness.waitForTaskRunning();

	final int numRecords1 = 5;
	final int numRecords2 = 3;

	for (int x = 0; x < numRecords1; x++) {
		testHarness.processElement(new StreamRecord<>("hello"), 0, 0);
	}

	for (int x = 0; x < numRecords2; x++) {
		testHarness.processElement(new StreamRecord<>("hello"), 1, 0);
	}
	testHarness.waitForInputProcessing();

	assertEquals(numRecords1 + numRecords2, numRecordsInCounter.getCount());
	assertEquals((numRecords1 + numRecords2) * 2 * 2 * 2, numRecordsOutCounter.getCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

public TestTaskStateManager getTaskStateManager() {
	return taskStateManager;
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {

		ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
		PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
		Executor executor = mock(Executor.class);
		ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

		TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

		JobInformation jobInformation = new JobInformation(
				new JobID(),
				"Job Name",
				new SerializedValue<>(new ExecutionConfig()),
				new Configuration(),
				Collections.emptyList(),
				Collections.emptyList());

		TaskInformation taskInformation = new TaskInformation(
				new JobVertexID(),
				"Test Task",
				1,
				1,
				invokableClass.getName(),
				new Configuration());

		return new Task(
				jobInformation,
				taskInformation,
				new ExecutionAttemptID(),
				new AllocationID(),
				0,
				0,
				Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
				Collections.<InputGateDeploymentDescriptor>emptyList(),
				0,
				mock(MemoryManager.class),
				mock(IOManager.class),
				shuffleEnvironment,
				new KvStateService(new KvStateRegistry(), null, null),
				mock(BroadcastVariableManager.class),
				new TaskEventDispatcher(),
				ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
				new TestTaskStateManager(),
				mock(TaskManagerActions.class),
				mock(InputSplitProvider.class),
				mock(CheckpointResponder.class),
				new NoOpTaskOperatorEventGateway(),
				new TestGlobalAggregateManager(),
				TestingClassLoaderLease.newBuilder().build(),
				mock(FileCache.class),
				new TestingTaskManagerRuntimeInfo(),
				taskMetricGroup,
				consumableNotifier,
				partitionProducerStateChecker,
				executor);
	}
 

/**
 * This tests the forwarding of jm and tm-local state from the futures reported by the backends, through the
 * async checkpointing thread to the {@link org.apache.flink.runtime.state.TaskStateManager}.
 */
@Test
public void testReportingFromSnapshotToTaskStateManager() throws Exception {

	TestTaskStateManager taskStateManager = new TestTaskStateManager();

	StreamMockEnvironment streamMockEnvironment = new StreamMockEnvironment(
		new Configuration(),
		new Configuration(),
		new ExecutionConfig(),
		1024 * 1024,
		new MockInputSplitProvider(),
		0,
		taskStateManager);

	StreamTask testStreamTask = new StreamTaskTest.NoOpStreamTask(streamMockEnvironment);
	CheckpointMetaData checkpointMetaData = new CheckpointMetaData(0L, 0L);
	CheckpointMetrics checkpointMetrics = new CheckpointMetrics();

	Map<OperatorID, OperatorSnapshotFutures> snapshots = new HashMap<>(1);
	OperatorSnapshotFutures osFuture = new OperatorSnapshotFutures();

	osFuture.setKeyedStateManagedFuture(createSnapshotResult(KeyedStateHandle.class));
	osFuture.setKeyedStateRawFuture(createSnapshotResult(KeyedStateHandle.class));
	osFuture.setOperatorStateManagedFuture(createSnapshotResult(OperatorStateHandle.class));
	osFuture.setOperatorStateRawFuture(createSnapshotResult(OperatorStateHandle.class));
	osFuture.setInputChannelStateFuture(createSnapshotCollectionResult(InputChannelStateHandle.class));
	osFuture.setResultSubpartitionStateFuture(createSnapshotCollectionResult(ResultSubpartitionStateHandle.class));

	OperatorID operatorID = new OperatorID();
	snapshots.put(operatorID, osFuture);

	AsyncCheckpointRunnable checkpointRunnable = new AsyncCheckpointRunnable(
		snapshots,
		checkpointMetaData,
		checkpointMetrics,
		0L,
		testStreamTask.getName(),
		asyncCheckpointRunnable -> {},
		asyncCheckpointRunnable -> {},
		testStreamTask.getEnvironment(),
		testStreamTask);

	checkpointRunnable.run();

	TaskStateSnapshot lastJobManagerTaskStateSnapshot = taskStateManager.getLastJobManagerTaskStateSnapshot();
	TaskStateSnapshot lastTaskManagerTaskStateSnapshot = taskStateManager.getLastTaskManagerTaskStateSnapshot();

	OperatorSubtaskState jmState =
		lastJobManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);

	OperatorSubtaskState tmState =
		lastTaskManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);

	performCheck(osFuture.getKeyedStateManagedFuture(), jmState.getManagedKeyedState(), tmState.getManagedKeyedState());
	performCheck(osFuture.getKeyedStateRawFuture(), jmState.getRawKeyedState(), tmState.getRawKeyedState());
	performCheck(osFuture.getOperatorStateManagedFuture(), jmState.getManagedOperatorState(), tmState.getManagedOperatorState());
	performCheck(osFuture.getOperatorStateRawFuture(), jmState.getRawOperatorState(), tmState.getRawOperatorState());
	performCollectionCheck(osFuture.getInputChannelStateFuture(), jmState.getInputChannelState(), tmState.getInputChannelState());
	performCollectionCheck(osFuture.getResultSubpartitionStateFuture(), jmState.getResultSubpartitionState(), tmState.getResultSubpartitionState());
}
 

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	OneInputStreamOperator<String, String> headOperator = new WatermarkMetricOperator();
	OperatorID headOperatorId = new OperatorID();

	OneInputStreamOperator<String, String> chainedOperator = new WatermarkMetricOperator();
	OperatorID chainedOperatorId = new OperatorID();

	testHarness.setupOperatorChain(headOperatorId, headOperator)
		.chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
	InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
	InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {
		@Override
		public OperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
			if (id.equals(headOperatorId)) {
				return headOperatorMetricGroup;
			} else if (id.equals(chainedOperatorId)) {
				return chainedOperatorMetricGroup;
			} else {
				return super.getOrAddOperator(id, name);
			}
		}
	};

	StreamMockEnvironment env = new StreamMockEnvironment(
		testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {
		@Override
		public TaskMetricGroup getMetricGroup() {
			return taskMetricGroup;
		}
	};

	testHarness.invoke(env);
	testHarness.waitForTaskRunning();

	Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
	Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);

	Assert.assertEquals("A metric was registered multiple times.",
		5,
		new HashSet<>(Arrays.asList(
			taskInputWatermarkGauge,
			headInputWatermarkGauge,
			headOutputWatermarkGauge,
			chainedInputWatermarkGauge,
			chainedOutputWatermarkGauge))
			.size());

	Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.processElement(new Watermark(1L));
	testHarness.waitForInputProcessing();
	Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.processElement(new Watermark(2L));
	testHarness.waitForInputProcessing();
	Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(8L, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}