org.apache.flink.runtime.executiongraph.ErrorInfo Java Examples

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void failureInfoIsSetAfterTaskFailure() {
	final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
	final JobID jobId = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices());
	final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();

	final String exceptionMessage = "expected exception";
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, attemptId, ExecutionState.FAILED, new RuntimeException(exceptionMessage)));

	final ErrorInfo failureInfo = scheduler.requestJob().getFailureInfo();
	assertThat(failureInfo, is(notNullValue()));
	assertThat(failureInfo.getExceptionAsString(), containsString(exceptionMessage));
}
 

public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"DefaultExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null,
		"stateBackendName");

	jobStatus = super.getState();
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		// the failure cause in the execution graph should not be the cause of the canceled job result
		assertThat(expected.getCause(), is(nullValue()));
	}
}
 

/**
 * Test that {@link JobResult} is cached when the job finishes.
 */
@Test
public void testCacheJobExecutionResult() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	final JobID failedJobId = new JobID();

	final JobStatus expectedState = JobStatus.FAILED;
	final ArchivedExecutionGraph failedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(failedJobId)
		.setState(expectedState)
		.setFailureCause(new ErrorInfo(new RuntimeException("expected"), 1L))
		.build();

	dispatcher.completeJobExecution(failedExecutionGraph);

	assertThat(
		dispatcherGateway.requestJobStatus(failedJobId, TIMEOUT).get(),
		equalTo(expectedState));
	assertThat(
		dispatcherGateway.requestJob(failedJobId, TIMEOUT).get(),
		equalTo(failedExecutionGraph));
}
 

public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"ExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null);

	jobStatus = super.getState();
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void testCancelledJobThrowsJobCancellationException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.CANCELED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with an JobCancellationException.");
	} catch (JobCancellationException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

@Test
public void testFailedJobThrowsJobExecutionException() throws Exception {
	final FlinkException cause = new FlinkException("Test exception");
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(cause, 42L))
			.build());

	try {
		jobResult.toJobExecutionResult(getClass().getClassLoader());
		fail("Job should fail with JobExecutionException.");
	} catch (JobExecutionException expected) {
		assertThat(expected.getCause(), is(equalTo(cause)));
	}
}
 

public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"ExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null);

	jobStatus = super.getState();
}
 

/**
 * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which
 * must be in a globally terminal state.
 *
 * @param accessExecutionGraph to create the JobResult from
 * @return JobResult of the given AccessExecutionGraph
 */
public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) {
	final JobID jobId = accessExecutionGraph.getJobID();
	final JobStatus jobStatus = accessExecutionGraph.getState();

	checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " +
			"terminal state. It is in state " + jobStatus + '.');

	final JobResult.Builder builder = new JobResult.Builder();
	builder.jobId(jobId);

	builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState()));

	final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED);
	// guard against clock changes
	final long guardedNetRuntime = Math.max(netRuntime, 0L);
	builder.netRuntime(guardedNetRuntime);
	builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());

	if (jobStatus == JobStatus.FAILED) {
		final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();
		checkNotNull(errorInfo, "No root cause is found for the job failure.");

		builder.serializedThrowable(errorInfo.getException());
	}

	return builder.build();
}
 

private static JobExceptionsInfo createJobExceptionsInfo(AccessExecutionGraph executionGraph) {
	ErrorInfo rootException = executionGraph.getFailureInfo();
	String rootExceptionMessage = null;
	Long rootTimestamp = null;
	if (rootException != null) {
		rootExceptionMessage = rootException.getExceptionAsString();
		rootTimestamp = rootException.getTimestamp();
	}

	List<JobExceptionsInfo.ExecutionExceptionInfo> taskExceptionList = new ArrayList<>();
	boolean truncated = false;
	for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) {
		String t = task.getFailureCauseAsString();
		if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
			if (taskExceptionList.size() >= MAX_NUMBER_EXCEPTION_TO_REPORT) {
				truncated = true;
				break;
			}

			TaskManagerLocation location = task.getCurrentAssignedResourceLocation();
			String locationString = location != null ?
				location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)";
			long timestamp = task.getStateTimestamp(ExecutionState.FAILED);
			taskExceptionList.add(new JobExceptionsInfo.ExecutionExceptionInfo(
				t,
				task.getTaskNameWithSubtaskIndex(),
				locationString,
				timestamp == 0 ? -1 : timestamp));
		}
	}

	return new JobExceptionsInfo(rootExceptionMessage, rootTimestamp, taskExceptionList, truncated);
}
 

/**
 * Test that {@link JobResult} is cached when the job finishes.
 */
@Test
public void testCacheJobExecutionResult() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	final JobID failedJobId = new JobID();

	final JobStatus expectedState = JobStatus.FAILED;
	final ArchivedExecutionGraph failedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(failedJobId)
		.setState(expectedState)
		.setFailureCause(new ErrorInfo(new RuntimeException("expected"), 1L))
		.build();

	dispatcher.completeJobExecution(failedExecutionGraph);

	assertThat(
		dispatcherGateway.requestJobStatus(failedJobId, TIMEOUT).get(),
		equalTo(expectedState));
	assertThat(
		dispatcherGateway.requestJob(failedJobId, TIMEOUT).get(),
		equalTo(failedExecutionGraph));
}
 

@Test
public void testFailedJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(new FlinkException("Test exception"), 42L))
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

@Test
public void testFailedJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(new FlinkException("Test exception"), 42L))
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

private static JobExceptionsInfo createJobExceptionsInfo(AccessExecutionGraph executionGraph, int exceptionToReportMaxSize) {
	ErrorInfo rootException = executionGraph.getFailureInfo();
	String rootExceptionMessage = null;
	Long rootTimestamp = null;
	if (rootException != null) {
		rootExceptionMessage = rootException.getExceptionAsString();
		rootTimestamp = rootException.getTimestamp();
	}

	List<JobExceptionsInfo.ExecutionExceptionInfo> taskExceptionList = new ArrayList<>();
	boolean truncated = false;
	for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) {
		String t = task.getFailureCauseAsString();
		if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
			if (taskExceptionList.size() >= exceptionToReportMaxSize) {
				truncated = true;
				break;
			}

			TaskManagerLocation location = task.getCurrentAssignedResourceLocation();
			String locationString = location != null ?
				location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)";
			long timestamp = task.getStateTimestamp(ExecutionState.FAILED);
			taskExceptionList.add(new JobExceptionsInfo.ExecutionExceptionInfo(
				t,
				task.getTaskNameWithSubtaskIndex(),
				locationString,
				timestamp == 0 ? -1 : timestamp));
		}
	}

	return new JobExceptionsInfo(rootExceptionMessage, rootTimestamp, taskExceptionList, truncated);
}
 

private static void generateArchivedJob() throws Exception {
	// Attempt
	StringifiedAccumulatorResult acc1 = new StringifiedAccumulatorResult("name1", "type1", "value1");
	StringifiedAccumulatorResult acc2 = new StringifiedAccumulatorResult("name2", "type2", "value2");
	TaskManagerLocation location = new TaskManagerLocation(new ResourceID("hello"), InetAddress.getLocalHost(), 1234);
	AllocationID allocationID = new AllocationID(42L, 43L);
	originalAttempt = new ArchivedExecutionBuilder()
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9})
		.setParallelSubtaskIndex(1)
		.setAttemptNumber(0)
		.setAssignedResourceLocation(location)
		.setAssignedAllocationID(allocationID)
		.setUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.setState(ExecutionState.FINISHED)
		.setFailureCause("attemptException")
		.build();
	// Subtask
	originalSubtask = new ArchivedExecutionVertexBuilder()
		.setSubtaskIndex(originalAttempt.getParallelSubtaskIndex())
		.setTaskNameWithSubtask("hello(1/1)")
		.setCurrentExecution(originalAttempt)
		.build();
	// Task
	originalTask = new ArchivedExecutionJobVertexBuilder()
		.setTaskVertices(new ArchivedExecutionVertex[]{originalSubtask})
		.build();
	// Job
	Map<JobVertexID, ArchivedExecutionJobVertex> tasks = new HashMap<>();
	tasks.put(originalTask.getJobVertexId(), originalTask);
	originalJob = new ArchivedExecutionGraphBuilder()
		.setJobID(new JobID())
		.setTasks(tasks)
		.setFailureCause(new ErrorInfo(new Exception("jobException"), originalAttempt.getStateTimestamp(ExecutionState.FAILED)))
		.setState(JobStatus.FINISHED)
		.setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
		.setArchivedUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2})
		.build();
}
 

/**
 * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which
 * must be in a globally terminal state.
 *
 * @param accessExecutionGraph to create the JobResult from
 * @return JobResult of the given AccessExecutionGraph
 */
public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) {
	final JobID jobId = accessExecutionGraph.getJobID();
	final JobStatus jobStatus = accessExecutionGraph.getState();

	checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " +
			"terminal state. It is in state " + jobStatus + '.');

	final JobResult.Builder builder = new JobResult.Builder();
	builder.jobId(jobId);

	builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState()));

	final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED);
	// guard against clock changes
	final long guardedNetRuntime = Math.max(netRuntime, 0L);
	builder.netRuntime(guardedNetRuntime);
	builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());

	if (jobStatus != JobStatus.FINISHED) {
		final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();

		if (errorInfo != null) {
			builder.serializedThrowable(errorInfo.getException());
		}
	}

	return builder.build();
}
 

@Test
public void testFailedJobIsFailureResult() {
	final JobResult jobResult = JobResult.createFrom(
		new ArchivedExecutionGraphBuilder()
			.setJobID(new JobID())
			.setState(JobStatus.FAILED)
			.setFailureCause(new ErrorInfo(new FlinkException("Test exception"), 42L))
			.build());

	assertThat(jobResult.isSuccess(), is(false));
}
 

/**
 * Test that {@link JobResult} is cached when the job finishes.
 */
@Test
public void testCacheJobExecutionResult() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	final JobID failedJobId = new JobID();

	final JobStatus expectedState = JobStatus.FAILED;
	final ArchivedExecutionGraph failedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(failedJobId)
		.setState(expectedState)
		.setFailureCause(new ErrorInfo(new RuntimeException("expected"), 1L))
		.build();

	dispatcher.completeJobExecution(failedExecutionGraph);

	assertThat(
		dispatcherGateway.requestJobStatus(failedJobId, TIMEOUT).get(),
		equalTo(expectedState));
	assertThat(
		dispatcherGateway.requestJob(failedJobId, TIMEOUT).get(),
		equalTo(failedExecutionGraph));
}
 

private static JobExceptionsInfo createJobExceptionsInfo(AccessExecutionGraph executionGraph) {
	ErrorInfo rootException = executionGraph.getFailureInfo();
	String rootExceptionMessage = null;
	Long rootTimestamp = null;
	if (rootException != null) {
		rootExceptionMessage = rootException.getExceptionAsString();
		rootTimestamp = rootException.getTimestamp();
	}

	List<JobExceptionsInfo.ExecutionExceptionInfo> taskExceptionList = new ArrayList<>();
	boolean truncated = false;
	for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) {
		String t = task.getFailureCauseAsString();
		if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
			if (taskExceptionList.size() >= MAX_NUMBER_EXCEPTION_TO_REPORT) {
				truncated = true;
				break;
			}

			TaskManagerLocation location = task.getCurrentAssignedResourceLocation();
			String locationString = location != null ?
				location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)";
			long timestamp = task.getStateTimestamp(ExecutionState.FAILED);
			taskExceptionList.add(new JobExceptionsInfo.ExecutionExceptionInfo(
				t,
				task.getTaskNameWithSubtaskIndex(),
				locationString,
				timestamp == 0 ? -1 : timestamp));
		}
	}

	return new JobExceptionsInfo(rootExceptionMessage, rootTimestamp, taskExceptionList, truncated);
}
 

/**
 * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which
 * must be in a globally terminal state.
 *
 * @param accessExecutionGraph to create the JobResult from
 * @return JobResult of the given AccessExecutionGraph
 */
public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) {
	final JobID jobId = accessExecutionGraph.getJobID();
	final JobStatus jobStatus = accessExecutionGraph.getState();

	checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " +
			"terminal state. It is in state " + jobStatus + '.');

	final JobResult.Builder builder = new JobResult.Builder();
	builder.jobId(jobId);

	builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState()));

	final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED);
	// guard against clock changes
	final long guardedNetRuntime = Math.max(netRuntime, 0L);
	builder.netRuntime(guardedNetRuntime);
	builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());

	if (jobStatus != JobStatus.FINISHED) {
		final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();

		if (errorInfo != null) {
			builder.serializedThrowable(errorInfo.getException());
		}
	}

	return builder.build();
}
 

public ArchivedExecutionGraphBuilder setFailureCause(ErrorInfo failureCause) {
	this.failureCause = failureCause;
	return this;
}
 

public ArchivedExecutionGraphBuilder setFailureCause(ErrorInfo failureCause) {
	this.failureCause = failureCause;
	return this;
}
 

public ArchivedExecutionGraphBuilder setFailureCause(ErrorInfo failureCause) {
	this.failureCause = failureCause;
	return this;
}