Java Code Examples for org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness#getTask()

The following examples show how to use org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness#getTask() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StatefulOperatorChainedTaskTest.java From Flink-CEPplus with Apache License 2.0

4 votes

private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}

Example 2

Source File: StreamTaskTimerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testOpenCloseAndTimestamps() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	// first one spawns thread
	mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
		@Override
		public void onProcessingTime(long timestamp) {
		}
	});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}

Example 3

Source File: StreamTaskTimerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void checkScheduledTimestampe() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	final AtomicReference<Throwable> errorRef = new AtomicReference<>();

	final long t1 = System.currentTimeMillis();
	final long t2 = System.currentTimeMillis() - 200;
	final long t3 = System.currentTimeMillis() + 100;
	final long t4 = System.currentTimeMillis() + 200;

	ProcessingTimeService timeService = mapTask.getProcessingTimeService();
	timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
	timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
	timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
	timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));

	long deadline = System.currentTimeMillis() + 20000;
	while (errorRef.get() == null &&
			ValidatingProcessingTimeCallback.numInSequence < 4 &&
			System.currentTimeMillis() < deadline) {
		Thread.sleep(100);
	}

	// handle errors
	if (errorRef.get() != null) {
		errorRef.get().printStackTrace();
		fail(errorRef.get().getMessage());
	}

	assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// wait until the trigger thread is shut down. otherwise, the other tests may become unstable
	deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}

Example 4

Source File: StatefulOperatorChainedTaskTest.java From flink with Apache License 2.0

4 votes

private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}

Example 5

Source File: StreamTaskTimerTest.java From flink with Apache License 2.0

4 votes

@Test
public void testOpenCloseAndTimestamps() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	// first one spawns thread
	mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
		@Override
		public void onProcessingTime(long timestamp) {
		}
	});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}

Example 6

Source File: StreamTaskTimerTest.java From flink with Apache License 2.0

4 votes

@Test
public void checkScheduledTimestampe() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	final AtomicReference<Throwable> errorRef = new AtomicReference<>();

	final long t1 = System.currentTimeMillis();
	final long t2 = System.currentTimeMillis() - 200;
	final long t3 = System.currentTimeMillis() + 100;
	final long t4 = System.currentTimeMillis() + 200;

	ProcessingTimeService timeService = mapTask.getProcessingTimeService();
	timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
	timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
	timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
	timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));

	long deadline = System.currentTimeMillis() + 20000;
	while (errorRef.get() == null &&
			ValidatingProcessingTimeCallback.numInSequence < 4 &&
			System.currentTimeMillis() < deadline) {
		Thread.sleep(100);
	}

	// handle errors
	if (errorRef.get() != null) {
		errorRef.get().printStackTrace();
		fail(errorRef.get().getMessage());
	}

	assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// wait until the trigger thread is shut down. otherwise, the other tests may become unstable
	deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}

Example 7

Source File: StatefulOperatorChainedTaskTest.java From flink with Apache License 2.0

4 votes

private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}