org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo Java Examples

The following examples show how to use org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected BinaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double) perSortMemory / totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	
	this.inputs = new ArrayList<>();
	this.comparators = new ArrayList<>();
	this.sorters = new ArrayList<>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected BinaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double) perSortMemory / totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	
	this.inputs = new ArrayList<>();
	this.comparators = new ArrayList<>();
	this.sorters = new ArrayList<>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #11
Source Project: flink   Author: apache   File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #12
Source Project: flink   Author: apache   File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #13
Source Project: flink   Author: apache   File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected BinaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double) perSortMemory / totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? MemoryManagerBuilder.newBuilder().setMemorySize(totalMem).build() : null;
	
	this.inputs = new ArrayList<>();
	this.comparators = new ArrayList<>();
	this.sorters = new ArrayList<>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #14
Source Project: flink   Author: apache   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #15
Source Project: flink   Author: apache   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
static Environment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	Environment env = mock(Environment.class);
	when(env.getJobID()).thenReturn(new JobID());
	when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
	when(env.getIOManager()).thenReturn(ioMan);
	when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

	TaskInfo taskInfo = mock(TaskInfo.class);
	when(env.getTaskInfo()).thenReturn(taskInfo);
	when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);

	TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
	when(env.getTaskManagerInfo()).thenReturn(tmInfo);

	TestTaskStateManager taskStateManager = new TestTaskStateManager();
	when(env.getTaskStateManager()).thenReturn(taskStateManager);

	return env;
}
 
Example #17
Source Project: Flink-CEPplus   Author: ljygz   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, new ExecutionConfig(), env, timeProvider);
	});
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
static Environment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	Environment env = mock(Environment.class);
	when(env.getJobID()).thenReturn(new JobID());
	when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
	when(env.getIOManager()).thenReturn(ioMan);
	when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

	TaskInfo taskInfo = mock(TaskInfo.class);
	when(env.getTaskInfo()).thenReturn(taskInfo);
	when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);

	TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
	when(env.getTaskManagerInfo()).thenReturn(tmInfo);

	TestTaskStateManager taskStateManager = new TestTaskStateManager();
	when(env.getTaskStateManager()).thenReturn(taskStateManager);

	return env;
}
 
Example #19
Source Project: flink   Author: flink-tpc-ds   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, new ExecutionConfig(), env, timeProvider);
	});
}
 
Example #20
Source Project: flink   Author: apache   File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
static MockEnvironment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	return MockEnvironment.builder()
		.setUserCodeClassLoader(RocksDBStateBackendConfigTest.class.getClassLoader())
		.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings))
		.setIOManager(ioMan).build();
}
 
Example #21
Source Project: flink   Author: apache   File: TestTaskContext.java    License: Apache License 2.0 5 votes vote down vote up
public TestTaskContext(long memoryInBytes) {
	this.memoryManager = MemoryManagerBuilder
		.newBuilder()
		.setMemorySize(memoryInBytes)
		.build();
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #22
Source Project: flink   Author: apache   File: StreamSourceOperatorLatencyMetricsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that latency metrics can be enabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, new ExecutionConfig(), env, timeProvider);
	});
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: StatefulOperatorChainedTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
private JobManagerTaskRestore createRunAndCheckpointOperatorChain(
	OperatorID headId,
	OneInputStreamOperator<String, String> headOperator,
	OperatorID tailId,
	OneInputStreamOperator<String, String> tailOperator,
	Optional<JobManagerTaskRestore> restore) throws Exception {

	File localRootDir = temporaryFolder.newFolder();
	final OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO,
			localRootDir);

	testHarness.setupOperatorChain(headId, headOperator)
		.chain(tailId, tailOperator, StringSerializer.INSTANCE, true)
		.finish();

	if (restore.isPresent()) {
		JobManagerTaskRestore taskRestore = restore.get();
		testHarness.setTaskStateSnapshot(
			taskRestore.getRestoreCheckpointId(),
			taskRestore.getTaskStateSnapshot());
	}

	StreamMockEnvironment environment = new StreamMockEnvironment(
		testHarness.jobConfig,
		testHarness.taskConfig,
		testHarness.getExecutionConfig(),
		testHarness.memorySize,
		new MockInputSplitProvider(),
		testHarness.bufferSize,
		testHarness.getTaskStateManager());

	Configuration configuration = new Configuration();
	configuration.setString(STATE_BACKEND.key(), "rocksdb");
	File file = temporaryFolder.newFolder();
	configuration.setString(CHECKPOINTS_DIRECTORY.key(), file.toURI().toString());
	configuration.setString(INCREMENTAL_CHECKPOINTS.key(), "true");
	environment.setTaskManagerInfo(
		new TestingTaskManagerRuntimeInfo(
			configuration,
			System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator)));
	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	OneInputStreamTask<String, String> streamTask = testHarness.getTask();

	processRecords(testHarness);
	triggerCheckpoint(testHarness, streamTask);

	TestTaskStateManager taskStateManager = testHarness.getTaskStateManager();

	JobManagerTaskRestore jobManagerTaskRestore = new JobManagerTaskRestore(
		taskStateManager.getReportedCheckpointId(),
		taskStateManager.getLastJobManagerTaskStateSnapshot());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	return jobManagerTaskRestore;
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: TaskAsyncCallTest.java    License: Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(new TestUserCodeClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
	NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
	when(networkEnvironment.getResultPartitionManager()).thenReturn(partitionManager);
	when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: TaskTest.java    License: Apache License 2.0 4 votes vote down vote up
private Task build() throws Exception {
	final JobID jobId = new JobID();
	final JobVertexID jobVertexId = new JobVertexID();
	final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		new Configuration());

	final BlobCacheService blobCacheService = new BlobCacheService(
		mock(PermanentBlobCache.class),
		mock(TransientBlobCache.class));

	final TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		new AllocationID(),
		0,
		0,
		Collections.emptyList(),
		Collections.emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new TestGlobalAggregateManager(),
		blobCacheService,
		libraryCacheManager,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: TestTaskContext.java    License: Apache License 2.0 4 votes vote down vote up
public TestTaskContext(long memoryInBytes) {
	this.memoryManager = new MemoryManager(memoryInBytes, 1, 32 * 1024, MemoryType.HEAP, true);
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: DummyEnvironment.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskManagerRuntimeInfo getTaskManagerInfo() {
	return new TestingTaskManagerRuntimeInfo();
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: StreamTaskTerminationTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class);
	when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		networkEnv,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: StreamTaskTest.java    License: Apache License 2.0 4 votes vote down vote up
public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();

	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.getResultPartitionManager()).thenReturn(partitionManager);
	when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		network,
		mock(BroadcastVariableManager.class),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #30
Source Project: Flink-CEPplus   Author: ljygz   File: TaskCheckpointingBehaviourTest.java    License: Apache License 2.0 4 votes vote down vote up
private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder,
	boolean failOnCheckpointErrors) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setFailTaskOnCheckpointError(failOnCheckpointErrors);

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	TaskKvStateRegistry mockKvRegistry = mock(TaskKvStateRegistry.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry);
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			network,
			mock(BroadcastVariableManager.class),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}