org.apache.flink.runtime.operators.testutils.MockInputSplitProvider Java Examples

The following examples show how to use org.apache.flink.runtime.operators.testutils.MockInputSplitProvider. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {
	this(
		new JobID(),
		new ExecutionAttemptID(0L, 0L),
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		inputSplitProvider,
		bufferSize,
		taskStateManager);
}
 
Example #2
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
public AbstractStreamOperatorTestHarness(
		StreamOperatorFactory<OUT> factory,
		int maxParallelism,
		int parallelism,
		int subtaskIndex,
		OperatorID operatorID) throws Exception {
	this(
			null,
			factory,
			new MockEnvironmentBuilder()
					.setTaskName("MockTask")
					.setManagedMemorySize(3 * 1024 * 1024)
					.setInputSplitProvider(new MockInputSplitProvider())
					.setBufferSize(1024)
					.setMaxParallelism(maxParallelism)
					.setParallelism(parallelism)
					.setSubtaskIndex(subtaskIndex)
					.build(),
			true,
			operatorID);
}
 
Example #3
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
public AbstractStreamOperatorTestHarness(
		StreamOperator<OUT> operator,
		int maxParallelism,
		int parallelism,
		int subtaskIndex,
		OperatorID operatorID) throws Exception {
	this(
			operator,
			SimpleOperatorFactory.of(operator),
			new MockEnvironmentBuilder()
					.setTaskName("MockTask")
					.setManagedMemorySize(3 * 1024 * 1024)
					.setInputSplitProvider(new MockInputSplitProvider())
					.setBufferSize(1024)
					.setMaxParallelism(maxParallelism)
					.setParallelism(parallelism)
					.setSubtaskIndex(subtaskIndex)
					.build(),
			true,
			operatorID);
}
 
Example #4
Source File: SourceFunctionUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setManagedMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #5
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {
	this(
		new JobID(),
		new ExecutionAttemptID(0L, 0L),
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		inputSplitProvider,
		bufferSize,
		taskStateManager);
}
 
Example #6
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
public AbstractStreamOperatorTestHarness(
		StreamOperator<OUT> operator,
		int maxParallelism,
		int parallelism,
		int subtaskIndex,
		OperatorID operatorID) throws Exception {
	this(
		operator,
		new MockEnvironmentBuilder()
			.setTaskName("MockTask")
			.setMemorySize(3 * 1024 * 1024)
			.setInputSplitProvider(new MockInputSplitProvider())
			.setBufferSize(1024)
			.setMaxParallelism(maxParallelism)
			.setParallelism(parallelism)
			.setSubtaskIndex(subtaskIndex)
			.build(),
		true,
		operatorID);
}
 
Example #7
Source File: SourceFunctionUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #8
Source File: AbstractStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public AbstractStreamOperatorTestHarness(
		StreamOperator<OUT> operator,
		int maxParallelism,
		int parallelism,
		int subtaskIndex,
		OperatorID operatorID) throws Exception {
	this(
		operator,
		new MockEnvironmentBuilder()
			.setTaskName("MockTask")
			.setMemorySize(3 * 1024 * 1024)
			.setInputSplitProvider(new MockInputSplitProvider())
			.setBufferSize(1024)
			.setMaxParallelism(maxParallelism)
			.setParallelism(parallelism)
			.setSubtaskIndex(subtaskIndex)
			.build(),
		true,
		operatorID);
}
 
Example #9
Source File: SourceFunctionUtil.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #10
Source File: StreamMockEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {
	this(
		new JobID(),
		new ExecutionAttemptID(0L, 0L),
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		inputSplitProvider,
		bufferSize,
		taskStateManager);
}
 
Example #11
Source File: StreamMockEnvironment.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this(jobConfig, taskConfig, new ExecutionConfig(), memorySize, inputSplitProvider, bufferSize, taskStateManager);
}
 
Example #12
Source File: StreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment createEnvironment() {
	return new StreamMockEnvironment(
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		new MockInputSplitProvider(),
		bufferSize,
		taskStateManager);
}
 
Example #13
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment createEnvironment() {
	return new StreamMockEnvironment(
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		new MockInputSplitProvider(),
		bufferSize,
		taskStateManager);
}
 
Example #14
Source File: StreamOperatorChainingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private MockEnvironment createMockEnvironment(String taskName) {
	return new MockEnvironmentBuilder()
		.setTaskName(taskName)
		.setMemorySize(3 * 1024 * 1024)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(1024)
		.build();
}
 
Example #15
Source File: StreamTaskMailboxTestHarnessBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamTaskMailboxTestHarness<OUT> build() throws Exception {
	streamConfig.setBufferTimeout(bufferTimeout);

	TestTaskStateManager taskStateManager = new TestTaskStateManager(localRecoveryConfig);
	if (taskStateSnapshots != null) {
		taskStateManager.setReportedCheckpointId(taskStateSnapshots.keySet().iterator().next());
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(taskStateSnapshots);
	}

	StreamMockEnvironment streamMockEnvironment = new StreamMockEnvironment(
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		new MockInputSplitProvider(),
		bufferSize,
		taskStateManager);

	streamMockEnvironment.setCheckpointResponder(taskStateManager.getCheckpointResponder());
	initializeInputs(streamMockEnvironment);

	checkState(inputGates != null, "InputGates hasn't been initialised");

	StreamElementSerializer<OUT> outputStreamRecordSerializer = new StreamElementSerializer<>(outputSerializer);

	Queue<Object> outputList = new ArrayDeque<>();
	streamMockEnvironment.addOutput(outputList, outputStreamRecordSerializer);
	streamMockEnvironment.setTaskMetricGroup(taskMetricGroup);

	StreamTask<OUT, ?> task = taskFactory.apply(streamMockEnvironment);
	task.beforeInvoke();

	return new StreamTaskMailboxTestHarness<>(
		task,
		outputList,
		inputGates,
		streamMockEnvironment);
}
 
Example #16
Source File: StreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	BlockingCloseStreamOperator.resetLatches();
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {

		RunningTask<StreamTask<Void, BlockingCloseStreamOperator>> task = runTask(() -> new NoOpStreamTask<>(mockEnvironment));

		BlockingCloseStreamOperator.inClose.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(task.streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.finishClose.trigger();

		task.waitForTaskCompletion(false);

		// now the StreamTask should no longer be running
		assertFalse(task.streamTask.isRunning());
	}
}
 
Example #17
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this(jobConfig, taskConfig, new ExecutionConfig(), memorySize, inputSplitProvider, bufferSize, taskStateManager);
}
 
Example #18
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this.jobID = jobID;
	this.executionAttemptID = executionAttemptID;

	int subtaskIndex = 0;
	this.taskInfo = new TaskInfo(
		"", /* task name */
		1, /* num key groups / max parallelism */
		subtaskIndex, /* index of this subtask */
		1, /* num subtasks */
		0 /* attempt number */);
	this.jobConfiguration = jobConfig;
	this.taskConfiguration = taskConfig;
	this.inputs = new LinkedList<InputGate>();
	this.outputs = new LinkedList<ResultPartitionWriter>();
	this.memManager = new MemoryManager(memorySize, 1);
	this.ioManager = new IOManagerAsync();
	this.taskStateManager = Preconditions.checkNotNull(taskStateManager);
	this.aggregateManager = new TestGlobalAggregateManager();
	this.inputSplitProvider = inputSplitProvider;
	this.bufferSize = bufferSize;

	this.executionConfig = executionConfig;
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId());

	KvStateRegistry registry = new KvStateRegistry();
	this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
}
 
Example #19
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment createEnvironment() {
	return new StreamMockEnvironment(
		jobConfig,
		taskConfig,
		executionConfig,
		memorySize,
		new MockInputSplitProvider(),
		bufferSize,
		taskStateManager);
}
 
Example #20
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long offHeapMemorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this.jobID = jobID;
	this.executionAttemptID = executionAttemptID;

	int subtaskIndex = 0;
	this.taskInfo = new TaskInfo(
		"", /* task name */
		1, /* num key groups / max parallelism */
		subtaskIndex, /* index of this subtask */
		1, /* num subtasks */
		0 /* attempt number */);
	this.jobConfiguration = jobConfig;
	this.taskConfiguration = taskConfig;
	this.inputs = new LinkedList<>();
	this.outputs = new LinkedList<ResultPartitionWriter>();
	this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(offHeapMemorySize).build();
	this.ioManager = new IOManagerAsync();
	this.taskStateManager = Preconditions.checkNotNull(taskStateManager);
	this.aggregateManager = new TestGlobalAggregateManager();
	this.inputSplitProvider = inputSplitProvider;
	this.bufferSize = bufferSize;

	this.executionConfig = executionConfig;
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId());

	KvStateRegistry registry = new KvStateRegistry();
	this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
}
 
Example #21
Source File: AsyncWaitOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private MockEnvironment createMockEnvironment() {
	return new MockEnvironmentBuilder()
		.setTaskName("foobarTask")
		.setMemorySize(1024 * 1024L)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(4 * 1024)
		.build();
}
 
Example #22
Source File: StreamMockEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	Configuration jobConfig,
	Configuration taskConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this(jobConfig, taskConfig, new ExecutionConfig(), memorySize, inputSplitProvider, bufferSize, taskStateManager);
}
 
Example #23
Source File: StreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	BlockingCloseStreamOperator.resetLatches();
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setManagedMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {

		RunningTask<StreamTask<Void, BlockingCloseStreamOperator>> task = runTask(() -> new NoOpStreamTask<>(mockEnvironment));

		BlockingCloseStreamOperator.inClose.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(task.streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.finishClose.trigger();

		task.waitForTaskCompletion(false);

		// now the StreamTask should no longer be running
		assertFalse(task.streamTask.isRunning());
	}
}
 
Example #24
Source File: StreamOperatorChainingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private MockEnvironment createMockEnvironment(String taskName) {
	return new MockEnvironmentBuilder()
		.setTaskName(taskName)
		.setManagedMemorySize(3 * 1024 * 1024)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(1024)
		.build();
}
 
Example #25
Source File: StreamOperatorChainingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private MockEnvironment createMockEnvironment(String taskName) {
	return new MockEnvironmentBuilder()
		.setTaskName(taskName)
		.setMemorySize(3 * 1024 * 1024)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(1024)
		.build();
}
 
Example #26
Source File: AsyncWaitOperatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nonnull
private MockEnvironment createMockEnvironment() {
	return new MockEnvironmentBuilder()
		.setTaskName("foobarTask")
		.setMemorySize(1024 * 1024L)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(4 * 1024)
		.build();
}
 
Example #27
Source File: StreamMockEnvironment.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this.jobID = jobID;
	this.executionAttemptID = executionAttemptID;

	int subtaskIndex = 0;
	this.taskInfo = new TaskInfo(
		"", /* task name */
		1, /* num key groups / max parallelism */
		subtaskIndex, /* index of this subtask */
		1, /* num subtasks */
		0 /* attempt number */);
	this.jobConfiguration = jobConfig;
	this.taskConfiguration = taskConfig;
	this.inputs = new LinkedList<InputGate>();
	this.outputs = new LinkedList<ResultPartitionWriter>();
	this.memManager = new MemoryManager(memorySize, 1);
	this.ioManager = new IOManagerAsync();
	this.taskStateManager = Preconditions.checkNotNull(taskStateManager);
	this.aggregateManager = new TestGlobalAggregateManager();
	this.inputSplitProvider = inputSplitProvider;
	this.bufferSize = bufferSize;

	this.executionConfig = executionConfig;
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId());

	KvStateRegistry registry = new KvStateRegistry();
	this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
}
 
Example #28
Source File: LocalStateForwardingTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * This tests the forwarding of jm and tm-local state from the futures reported by the backends, through the
 * async checkpointing thread to the {@link org.apache.flink.runtime.state.TaskStateManager}.
 */
@Test
public void testReportingFromSnapshotToTaskStateManager() {

	TestTaskStateManager taskStateManager = new TestTaskStateManager();

	StreamMockEnvironment streamMockEnvironment = new StreamMockEnvironment(
		new Configuration(),
		new Configuration(),
		new ExecutionConfig(),
		1024 * 1024,
		new MockInputSplitProvider(),
		0,
		taskStateManager);

	StreamTask testStreamTask = new StreamTaskTest.NoOpStreamTask(streamMockEnvironment);
	CheckpointMetaData checkpointMetaData = new CheckpointMetaData(0L, 0L);
	CheckpointMetrics checkpointMetrics = new CheckpointMetrics();

	Map<OperatorID, OperatorSnapshotFutures> snapshots = new HashMap<>(1);
	OperatorSnapshotFutures osFuture = new OperatorSnapshotFutures();

	osFuture.setKeyedStateManagedFuture(createSnapshotResult(KeyedStateHandle.class));
	osFuture.setKeyedStateRawFuture(createSnapshotResult(KeyedStateHandle.class));
	osFuture.setOperatorStateManagedFuture(createSnapshotResult(OperatorStateHandle.class));
	osFuture.setOperatorStateRawFuture(createSnapshotResult(OperatorStateHandle.class));

	OperatorID operatorID = new OperatorID();
	snapshots.put(operatorID, osFuture);

	StreamTask.AsyncCheckpointRunnable checkpointRunnable =
		new StreamTask.AsyncCheckpointRunnable(
			testStreamTask,
			snapshots,
			checkpointMetaData,
			checkpointMetrics,
			0L);

	checkpointRunnable.run();

	TaskStateSnapshot lastJobManagerTaskStateSnapshot = taskStateManager.getLastJobManagerTaskStateSnapshot();
	TaskStateSnapshot lastTaskManagerTaskStateSnapshot = taskStateManager.getLastTaskManagerTaskStateSnapshot();

	OperatorSubtaskState jmState =
		lastJobManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);

	OperatorSubtaskState tmState =
		lastTaskManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);

	performCheck(osFuture.getKeyedStateManagedFuture(), jmState.getManagedKeyedState(), tmState.getManagedKeyedState());
	performCheck(osFuture.getKeyedStateRawFuture(), jmState.getRawKeyedState(), tmState.getRawKeyedState());
	performCheck(osFuture.getOperatorStateManagedFuture(), jmState.getManagedOperatorState(), tmState.getManagedOperatorState());
	performCheck(osFuture.getOperatorStateRawFuture(), jmState.getRawOperatorState(), tmState.getRawOperatorState());
}
 
Example #29
Source File: OneInputStreamTaskTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	OneInputStreamOperator<String, String> headOperator = new WatermarkMetricOperator();
	OperatorID headOperatorId = new OperatorID();

	OneInputStreamOperator<String, String> chainedOperator = new WatermarkMetricOperator();
	OperatorID chainedOperatorId = new OperatorID();

	testHarness.setupOperatorChain(headOperatorId, headOperator)
		.chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
	InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
	InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {
		@Override
		public OperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
			if (id.equals(headOperatorId)) {
				return headOperatorMetricGroup;
			} else if (id.equals(chainedOperatorId)) {
				return chainedOperatorMetricGroup;
			} else {
				return super.getOrAddOperator(id, name);
			}
		}
	};

	StreamMockEnvironment env = new StreamMockEnvironment(
		testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {
		@Override
		public TaskMetricGroup getMetricGroup() {
			return taskMetricGroup;
		}
	};

	testHarness.invoke(env);
	testHarness.waitForTaskRunning();

	Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
	Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
	Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);

	Assert.assertEquals("A metric was registered multiple times.",
		5,
		new HashSet<>(Arrays.asList(
			taskInputWatermarkGauge,
			headInputWatermarkGauge,
			headOutputWatermarkGauge,
			chainedInputWatermarkGauge,
			chainedOutputWatermarkGauge))
			.size());

	Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.processElement(new Watermark(1L));
	testHarness.waitForInputProcessing();
	Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.processElement(new Watermark(2L));
	testHarness.waitForInputProcessing();
	Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, headOutputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(4L, chainedInputWatermarkGauge.getValue().longValue());
	Assert.assertEquals(8L, chainedOutputWatermarkGauge.getValue().longValue());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 
Example #30
Source File: OneInputStreamTaskTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testOperatorMetricReuse() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOperatorChain(new OperatorID(), new DuplicatingOperator())
		.chain(new OperatorID(), new DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.chain(new OperatorID(), new DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	final TaskMetricGroup taskMetricGroup = new UnregisteredMetricGroups.UnregisteredTaskMetricGroup() {
		@Override
		public OperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) {
			return new OperatorMetricGroup(NoOpMetricRegistry.INSTANCE, this, operatorID, name);
		}
	};

	final StreamMockEnvironment env = new StreamMockEnvironment(
		testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {
		@Override
		public TaskMetricGroup getMetricGroup() {
			return taskMetricGroup;
		}
	};

	final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
	final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();

	testHarness.invoke(env);
	testHarness.waitForTaskRunning();

	final int numRecords = 5;

	for (int x = 0; x < numRecords; x++) {
		testHarness.processElement(new StreamRecord<>("hello"));
	}
	testHarness.waitForInputProcessing();

	assertEquals(numRecords, numRecordsInCounter.getCount());
	assertEquals(numRecords * 2 * 2 * 2, numRecordsOutCounter.getCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}