org.apache.flink.runtime.JobException Java Examples

The following examples show how to use org.apache.flink.runtime.JobException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ExecutionGraphTestUtils.java From flink with Apache License 2.0

6 votes

public ExecutionGraph build() throws JobException, JobExecutionException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		metricGroup,
		blobWriter,
		allocationTimeout,
		TEST_LOGGER,
		shuffleMaster,
		partitionTracker,
		failoverStrategyFactory);
}

Example #2

Source File: PipelinedFailoverRegionBuildingTest.java From flink with Apache License 2.0

6 votes

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}

Example #3

Source File: ExecutionJobVertexTest.java From flink with Apache License 2.0

6 votes

private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}

Example #4

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

/**
 * Check that channel and operator states are assigned to the same tasks on recovery.
 */
@Test
public void testChannelStateAssignmentStability() throws JobException, JobExecutionException {
	int numOperators = 10; // note: each operator is places into a separate vertex
	int numSubTasks = 100;

	Set<OperatorID> operatorIds = buildOperatorIds(numOperators);
	Map<OperatorID, OperatorState> states = buildOperatorStates(operatorIds, numSubTasks);
	Map<OperatorID, ExecutionJobVertex> vertices = buildVertices(operatorIds, numSubTasks);

	new StateAssignmentOperation(0, new HashSet<>(vertices.values()), states, false).assignStates();

	for (OperatorID operatorId : operatorIds) {
		for (int subtaskIdx = 0; subtaskIdx < numSubTasks; subtaskIdx++) {
			Assert.assertEquals(
				states.get(operatorId).getState(subtaskIdx),
				getAssignedState(vertices.get(operatorId), operatorId, subtaskIdx));
		}
	}
}

Example #5

Source File: LegacyScheduler.java From flink with Apache License 2.0

6 votes

private ExecutionGraph createExecutionGraph(
		JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
		ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws JobExecutionException, JobException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker);
}

Example #6

Source File: ExecutionJobVertex.java From flink with Apache License 2.0

6 votes

/**
 * Convenience constructor for testing.
 */
@VisibleForTesting
ExecutionJobVertex(
		ExecutionGraph graph,
		JobVertex jobVertex,
		int defaultParallelism,
		Time timeout) throws JobException {

	this(
		graph,
		jobVertex,
		defaultParallelism,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		timeout,
		1L,
		System.currentTimeMillis());
}

Example #7

Source File: ExecutionJobVertexTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}

Example #8

Source File: PipelinedFailoverRegionBuildingTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1000,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}

Example #9

Source File: TestingExecutionGraphBuilder.java From flink with Apache License 2.0

6 votes

public ExecutionGraph build() throws JobException, JobExecutionException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userClassLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		metricGroup,
		blobWriter,
		allocationTimeout,
		LOG,
		shuffleMaster,
		partitionTracker,
		failoverStrategyFactory);
}

Example #10

Source File: ExecutionGraphRestartTest.java From flink with Apache License 2.0

6 votes

private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	final ExecutionGraph executionGraph = new ExecutionGraph(
		new JobInformation(
			TEST_JOB_ID,
			"Test job",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList()),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		failoverStrategyFactory,
		slotProvider);

	executionGraph.start(mainThreadExecutor);
	executionGraph.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	return executionGraph;
}

Example #11

Source File: JobMaster.java From Flink-CEPplus with Apache License 2.0

6 votes

private ExecutionGraph createExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws JobExecutionException, JobException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration.getConfiguration(),
		scheduledExecutorService,
		scheduledExecutorService,
		scheduler,
		userCodeLoader,
		highAvailabilityServices.getCheckpointRecoveryFactory(),
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		jobMasterConfiguration.getSlotRequestTimeout(),
		log);
}

Example #12

Source File: ExecutionJobVertexTest.java From flink with Apache License 2.0

5 votes

private static ExecutionGraph createExecutionGraph() throws JobException, JobExecutionException {
	final ExecutionGraph executionGraph = TestingExecutionGraphBuilder
		.newBuilder()
		.setFutureExecutor(new DirectScheduledExecutorService())
		.build();
	executionGraph.transitionToRunning();
	return executionGraph;
}

Example #13

Source File: ExecutionGraphBuilder.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log)
	throws JobExecutionException, JobException {

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		-1,
		blobWriter,
		allocationTimeout,
		log);
}

Example #14

Source File: ExecutionVertex.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public void deployToSlot(LogicalSlot slot) throws JobException {
	if (currentExecution.tryAssignResource(slot)) {
		currentExecution.deploy();
	} else {
		throw new IllegalStateException("Could not assign resource " + slot + " to current execution " +
			currentExecution + '.');
	}
}

Example #15

Source File: ExecutionGraph.java From flink with Apache License 2.0

5 votes

public void scheduleForExecution() throws JobException {

		assertRunningInJobMasterMainThread();

		final long currentGlobalModVersion = globalModVersion;

		if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) {

			final CompletableFuture<Void> newSchedulingFuture = SchedulingUtils.schedule(
				scheduleMode,
				getAllExecutionVertices(),
				this);

			if (state == JobStatus.RUNNING && currentGlobalModVersion == globalModVersion) {
				schedulingFuture = newSchedulingFuture;
				newSchedulingFuture.whenComplete(
					(Void ignored, Throwable throwable) -> {
						if (throwable != null) {
							final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);

							if (!(strippedThrowable instanceof CancellationException)) {
								// only fail if the scheduling future was not canceled
								failGlobal(strippedThrowable);
							}
						}
					});
			} else {
				newSchedulingFuture.cancel(false);
			}
		}
		else {
			throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED);
		}
	}

Example #16

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test3NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(3 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(3, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
	}
}

Example #17

Source File: ExecutionGraphConstructionTest.java From flink with Apache License 2.0

5 votes

/**
 * Creates a JobGraph of the following form:
 * 
 * <pre>
 *  v1--->v2-->\
 *              \
 *               v4 --->\
 *        ----->/        \
 *  v3-->/                v5
 *       \               /
 *        ------------->/
 * </pre>
 */
@Test
public void testCreateSimpleGraphBipartite() throws Exception {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");
	
	v1.setParallelism(5);
	v2.setParallelism(7);
	v3.setParallelism(2);
	v4.setParallelism(11);
	v5.setParallelism(4);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);
	v5.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	
	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

	ExecutionGraph eg = createExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	verifyTestGraph(eg, v1, v2, v3, v4, v5);
}

Example #18

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNToN() throws Exception {
	final int N = 23;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex(), inEdges[0].getSource().getPartitionNumber());
	}
}

Example #19

Source File: ExecutionGraphRescalingTest.java From flink with Apache License 2.0

5 votes

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpJobMasterPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}

Example #20

Source File: ExecutionVertex.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public void deployToSlot(LogicalSlot slot) throws JobException {
	if (currentExecution.tryAssignResource(slot)) {
		currentExecution.deploy();
	} else {
		throw new IllegalStateException("Could not assign resource " + slot + " to current execution " +
			currentExecution + '.');
	}
}

Example #21

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test2NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(2 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(2, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 2, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 2 + 1, inEdges[1].getSource().getPartitionNumber());
	}
}

Example #22

Source File: ExecutionGraphDeploymentTest.java From flink with Apache License 2.0

5 votes

@Nonnull
private ExecutionGraph createExecutionGraphWithoutQueuedScheduling(
		JobID jobId,
		SlotProvider slotProvider,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor) throws JobException, JobExecutionException {
	return new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobId)
		.setFutureExecutor(futureExecutor)
		.setIoExecutor(ioExecutor)
		.setSlotProvider(slotProvider)
		.setBlobWriter(blobWriter)
		.setAllowQueuedScheduling(false)
		.build();
}

Example #23

Source File: ExecutionGraphRestartTest.java From flink with Apache License 2.0

5 votes

private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	return createSimpleExecutionGraph(restartStrategy, new RestartAllStrategy.Factory(), slotProvider, jobGraph);
}

Example #24

Source File: ExecutionGraph.java From flink with Apache License 2.0

5 votes

public void scheduleForExecution() throws JobException {

		assertRunningInJobMasterMainThread();

		if (isLegacyScheduling()) {
			LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
		}

		final long currentGlobalModVersion = globalModVersion;

		if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) {

			final CompletableFuture<Void> newSchedulingFuture = SchedulingUtils.schedule(
				scheduleMode,
				getAllExecutionVertices(),
				this);

			if (state == JobStatus.RUNNING && currentGlobalModVersion == globalModVersion) {
				schedulingFuture = newSchedulingFuture;
				newSchedulingFuture.whenComplete(
					(Void ignored, Throwable throwable) -> {
						if (throwable != null) {
							final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);

							if (!(strippedThrowable instanceof CancellationException)) {
								// only fail if the scheduling future was not canceled
								failGlobal(strippedThrowable);
							}
						}
					});
			} else {
				newSchedulingFuture.cancel(false);
			}
		}
		else {
			throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED);
		}
	}

Example #25

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNToN() throws Exception {
	final int N = 23;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex(), inEdges[0].getSource().getPartitionNumber());
	}
}

Example #26

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test2NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(2 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(2, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 2, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 2 + 1, inEdges[1].getSource().getPartitionNumber());
	}
}

Example #27

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test3NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(3 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(3, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
	}
}

Example #28

Source File: SchedulerBase.java From flink with Apache License 2.0

5 votes

private ExecutionGraph createExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy = legacyScheduling ?
		FailoverStrategyLoader.loadFailoverStrategy(jobMasterConfiguration, log) :
		new NoOpFailoverStrategy.Factory();

	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}

Example #29

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNTo7N() throws Exception {
	final int N = 11;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(7 * N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() / 7, inEdges[0].getSource().getPartitionNumber());
	}
}

Example #30

Source File: ExecutionGraphRescalingTest.java From flink with Apache License 2.0

5 votes

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}