Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils

The following examples show how to use org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
	onlyExecutionVertex.deployToSlot(testingLogicalSlot);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertTrue(taskManagerLocationOptional.isPresent());

	final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
	assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}
 
Example 2
/**
 * Tests that it will throw exception when getting the task manager location of a non existing execution.
 */
@Test
public void testGetNonExistingExecutionVertexWillThrowException() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID invalidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	try {
		inputsLocationsRetriever.getTaskManagerLocation(invalidExecutionVertexId);
		fail("Should throw exception if execution vertex doesn't exist!");
	} catch (IllegalStateException expected) {
		// expect this exception
	}
}
 
Example 3
/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
	onlyExecutionVertex.deployToSlot(testingLogicalSlot);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertTrue(taskManagerLocationOptional.isPresent());

	final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
	assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}
 
Example 4
/**
 * Tests that it will throw exception when getting the task manager location of a non existing execution.
 */
@Test
public void testGetNonExistingExecutionVertexWillThrowException() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID invalidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0);
	try {
		inputsLocationsRetriever.getTaskManagerLocation(invalidExecutionVertexId);
		fail("Should throw exception if execution vertex doesn't exist!");
	} catch (IllegalStateException expected) {
		// expect this exception
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: TaskExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
 
Example 6
private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	final Time timeout = Time.days(1L);

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);

	final ExecutionGraph executionGraph = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobVertex)
		.setRpcTimeout(timeout)
		.setAllocationTimeout(timeout)
		.allowQueuedScheduling()
		.build();

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);

	executionGraph.enableCheckpointing(
			chkConfig,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	return executionGraph;
}
 
Example 7
Source Project: flink   Source File: TaskExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
 
Example 8
/**
 * Tests that can get the producers of consumed result partitions.
 */
@Test
public void testGetConsumedResultPartitionsProducers() throws Exception {
	final JobVertex producer1 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex producer2 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex consumer = ExecutionGraphTestUtils.createNoOpVertex(1);
	consumer.connectNewDataSetAsInput(producer1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	consumer.connectNewDataSetAsInput(producer2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), producer1, producer2, consumer);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID evIdOfProducer1 = new ExecutionVertexID(producer1.getID(), 0);
	ExecutionVertexID evIdOfProducer2 = new ExecutionVertexID(producer2.getID(), 0);
	ExecutionVertexID evIdOfConsumer = new ExecutionVertexID(consumer.getID(), 0);

	Collection<Collection<ExecutionVertexID>> producersOfProducer1 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer1);
	Collection<Collection<ExecutionVertexID>> producersOfProducer2 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer2);
	Collection<Collection<ExecutionVertexID>> producersOfConsumer =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfConsumer);

	assertThat(producersOfProducer1, is(empty()));
	assertThat(producersOfProducer2, is(empty()));
	assertThat(producersOfConsumer, hasSize(2));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer1)));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer2)));
}
 
Example 9
/**
 * Tests that it will get empty task manager location if vertex is not scheduled.
 */
@Test
public void testGetEmptyTaskManagerLocationIfVertexNotScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocation =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertFalse(taskManagerLocation.isPresent());
}
 
Example 10
Source Project: flink   Source File: TaskExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private SupplierWithException<Boolean, Exception> jobIsRunning(Supplier<CompletableFuture<? extends AccessExecutionGraph>> executionGraphFutureSupplier) {
	final Predicate<AccessExecution> runningOrFinished = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.RUNNING).or(ExecutionGraphTestUtils.isInExecutionState(ExecutionState.FINISHED));
	final Predicate<AccessExecutionGraph> allExecutionsRunning = ExecutionGraphTestUtils.allExecutionsPredicate(runningOrFinished);

	return () -> {
		final AccessExecutionGraph executionGraph = executionGraphFutureSupplier.get().join();
		return allExecutionsRunning.test(executionGraph);
	};
}
 
Example 11
/**
 * Tests if the consumed inputs of the pipelined regions are computed
 * correctly using the Job graph below.
 * <pre>
 *          c
 *        /  X
 * a -+- b   e
 *       \  /
 *        d
 * </pre>
 * Pipelined regions: {a}, {b, c, d, e}
 */
@Test
public void returnsIncidentBlockingPartitions() throws Exception {
	final JobVertex a = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex b = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex c = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex d = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex e = ExecutionGraphTestUtils.createNoOpVertex(1);

	b.connectNewDataSetAsInput(a, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
	c.connectNewDataSetAsInput(b, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	d.connectNewDataSetAsInput(b, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	e.connectNewDataSetAsInput(c, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
	e.connectNewDataSetAsInput(d, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(a, b, c, d, e);
	final DefaultExecutionTopology topology = new DefaultExecutionTopology(simpleTestGraph);

	final DefaultSchedulingPipelinedRegion firstPipelinedRegion = topology.getPipelinedRegionOfVertex(new ExecutionVertexID(a.getID(), 0));
	final DefaultSchedulingPipelinedRegion secondPipelinedRegion = topology.getPipelinedRegionOfVertex(new ExecutionVertexID(e.getID(), 0));

	final DefaultExecutionVertex vertexB0 = topology.getVertex(new ExecutionVertexID(b.getID(), 0));
	final IntermediateResultPartitionID b0ConsumedResultPartition = Iterables.getOnlyElement(vertexB0.getConsumedResults()).getId();

	final Set<IntermediateResultPartitionID> secondPipelinedRegionConsumedResults = IterableUtils.toStream(secondPipelinedRegion.getConsumedResults())
		.map(DefaultResultPartition::getId)
		.collect(Collectors.toSet());

	assertThat(firstPipelinedRegion.getConsumedResults().iterator().hasNext(), is(false));
	assertThat(secondPipelinedRegionConsumedResults, contains(b0ConsumedResultPartition));
}
 
Example 12
/**
 * Tests that can get the producers of consumed result partitions.
 */
@Test
public void testGetConsumedResultPartitionsProducers() throws Exception {
	final JobVertex producer1 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex producer2 = ExecutionGraphTestUtils.createNoOpVertex(1);
	final JobVertex consumer = ExecutionGraphTestUtils.createNoOpVertex(1);
	consumer.connectNewDataSetAsInput(producer1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	consumer.connectNewDataSetAsInput(producer2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(producer1, producer2, consumer);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID evIdOfProducer1 = new ExecutionVertexID(producer1.getID(), 0);
	ExecutionVertexID evIdOfProducer2 = new ExecutionVertexID(producer2.getID(), 0);
	ExecutionVertexID evIdOfConsumer = new ExecutionVertexID(consumer.getID(), 0);

	Collection<Collection<ExecutionVertexID>> producersOfProducer1 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer1);
	Collection<Collection<ExecutionVertexID>> producersOfProducer2 =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfProducer2);
	Collection<Collection<ExecutionVertexID>> producersOfConsumer =
			inputsLocationsRetriever.getConsumedResultPartitionsProducers(evIdOfConsumer);

	assertThat(producersOfProducer1, is(empty()));
	assertThat(producersOfProducer2, is(empty()));
	assertThat(producersOfConsumer, hasSize(2));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer1)));
	assertThat(producersOfConsumer, hasItem(Collections.singletonList(evIdOfProducer2)));
}
 
Example 13
/**
 * Tests that it will get empty task manager location if vertex is not scheduled.
 */
@Test
public void testGetEmptyTaskManagerLocationIfVertexNotScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocation =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertFalse(taskManagerLocation.isPresent());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRequestNextInputSplit() throws Exception {
	final List<TestingInputSplit> expectedInputSplits = Arrays.asList(
		new TestingInputSplit(1),
		new TestingInputSplit(42),
		new TestingInputSplit(1337));

	// build one node JobGraph
	InputSplitSource<TestingInputSplit> inputSplitSource = new TestingInputSplitSource(expectedInputSplits);

	JobVertex source = new JobVertex("vertex1");
	source.setParallelism(1);
	source.setInputSplitSource(inputSplitSource);
	source.setInvokableClass(AbstractInvokable.class);

	final JobGraph testJobGraph = new JobGraph(source);
	testJobGraph.setAllowQueuedScheduling(true);

	configuration.setLong(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	configuration.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	final JobManagerSharedServices jobManagerSharedServices =
		new TestingJobManagerSharedServicesBuilder()
			.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
			.build();

	final JobMaster jobMaster = createJobMaster(
		configuration,
		testJobGraph,
		haServices,
		jobManagerSharedServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

		ExecutionGraph eg = jobMaster.getExecutionGraph();

		ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

		final SupplierWithException<SerializedInputSplit, Exception> inputSplitSupplier = () -> jobMasterGateway.requestNextInputSplit(
			source.getID(),
			ev.getCurrentExecutionAttempt().getAttemptId()).get();

		List<InputSplit> actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		final Matcher<Iterable<? extends InputSplit>> expectedInputSplitsMatcher = containsInAnyOrder(expectedInputSplits.toArray(EMPTY_TESTING_INPUT_SPLITS));
		assertThat(actualInputSplits, expectedInputSplitsMatcher);

		final long maxWaitMillis = 2000L;
		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		CompletableFuture.runAsync(() -> eg.failGlobal(new Exception("Testing exception")), eg.getJobMasterMainThreadExecutor()).get();

		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		assertThat(actualInputSplits, expectedInputSplitsMatcher);
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}