org.apache.flink.runtime.accumulators.AccumulatorSnapshot Java Examples

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

/**
 * Updates the accumulators during the runtime of a job. Final accumulator results are transferred
 * through the UpdateTaskExecutionState message.
 * @param accumulatorSnapshot The serialized flink and user-defined accumulators
 */
public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot) {
	Map<String, Accumulator<?, ?>> userAccumulators;
	try {
		userAccumulators = accumulatorSnapshot.deserializeUserAccumulators(userClassLoader);

		ExecutionAttemptID execID = accumulatorSnapshot.getExecutionAttemptID();
		Execution execution = currentExecutions.get(execID);
		if (execution != null) {
			execution.setAccumulators(userAccumulators);
		} else {
			LOG.debug("Received accumulator result for unknown execution {}.", execID);
		}
	} catch (Exception e) {
		LOG.error("Cannot update accumulators for job {}.", getJobID(), e);
	}
}
 

/**
 * Deserializes accumulators from a task state update.
 *
 * <p>This method never throws an exception!
 *
 * @param state The task execution state from which to deserialize the accumulators.
 * @return The deserialized accumulators, of null, if there are no accumulators or an error occurred.
 */
private Map<String, Accumulator<?, ?>> deserializeAccumulators(TaskExecutionState state) {
	AccumulatorSnapshot serializedAccumulators = state.getAccumulators();

	if (serializedAccumulators != null) {
		try {
			return serializedAccumulators.deserializeUserAccumulators(userClassLoader);
		}
		catch (Throwable t) {
			// we catch Throwable here to include all form of linking errors that may
			// occur if user classes are missing in the classpath
			LOG.error("Failed to deserialize final accumulator results.", t);
		}
	}
	return null;
}
 

@Override
public AccumulatorReport retrievePayload(ResourceID resourceID) {
	validateRunsInMainThread();
	return jobTable.getConnection(resourceID).map(
		jobManagerConnection -> {
			JobID jobId = jobManagerConnection.getJobId();

			List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16);
			Iterator<Task> allTasks = taskSlotTable.getTasks(jobId);

			while (allTasks.hasNext()) {
				Task task = allTasks.next();
				accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot());
			}
			return new AccumulatorReport(accumulatorSnapshots);
		}
	).orElseGet(() -> new AccumulatorReport(Collections.emptyList()));
}
 

@Override
public AccumulatorReport retrievePayload(ResourceID resourceID) {
	validateRunsInMainThread();
	JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID);
	if (jobManagerConnection != null) {
		JobID jobId = jobManagerConnection.getJobID();

		List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16);
		Iterator<Task> allTasks = taskSlotTable.getTasks(jobId);

		while (allTasks.hasNext()) {
			Task task = allTasks.next();
			accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot());
		}
		return new AccumulatorReport(accumulatorSnapshots);
	} else {
		return new AccumulatorReport(Collections.emptyList());
	}
}
 

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

/**
 * Deserializes accumulators from a task state update.
 *
 * <p>This method never throws an exception!
 *
 * @param state The task execution state from which to deserialize the accumulators.
 * @return The deserialized accumulators, of null, if there are no accumulators or an error occurred.
 */
private Map<String, Accumulator<?, ?>> deserializeAccumulators(TaskExecutionState state) {
	AccumulatorSnapshot serializedAccumulators = state.getAccumulators();

	if (serializedAccumulators != null) {
		try {
			return serializedAccumulators.deserializeUserAccumulators(userClassLoader);
		}
		catch (Throwable t) {
			// we catch Throwable here to include all form of linking errors that may
			// occur if user classes are missing in the classpath
			LOG.error("Failed to deserialize final accumulator results.", t);
		}
	}
	return null;
}
 

/**
 * Updates the accumulators during the runtime of a job. Final accumulator results are transferred
 * through the UpdateTaskExecutionState message.
 * @param accumulatorSnapshot The serialized flink and user-defined accumulators
 */
public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot) {
	Map<String, Accumulator<?, ?>> userAccumulators;
	try {
		userAccumulators = accumulatorSnapshot.deserializeUserAccumulators(userClassLoader);

		ExecutionAttemptID execID = accumulatorSnapshot.getExecutionAttemptID();
		Execution execution = currentExecutions.get(execID);
		if (execution != null) {
			execution.setAccumulators(userAccumulators);
		} else {
			LOG.debug("Received accumulator result for unknown execution {}.", execID);
		}
	} catch (Exception e) {
		LOG.error("Cannot update accumulators for job {}.", getJobID(), e);
	}
}
 

/**
 * Deserializes accumulators from a task state update.
 *
 * <p>This method never throws an exception!
 *
 * @param state The task execution state from which to deserialize the accumulators.
 * @return The deserialized accumulators, of null, if there are no accumulators or an error occurred.
 */
private Map<String, Accumulator<?, ?>> deserializeAccumulators(TaskExecutionState state) {
	AccumulatorSnapshot serializedAccumulators = state.getAccumulators();

	if (serializedAccumulators != null) {
		try {
			return serializedAccumulators.deserializeUserAccumulators(userClassLoader);
		}
		catch (Throwable t) {
			// we catch Throwable here to include all form of linking errors that may
			// occur if user classes are missing in the classpath
			LOG.error("Failed to deserialize final accumulator results.", t);
		}
	}
	return null;
}
 

/**
 * Updates the accumulators during the runtime of a job. Final accumulator results are transferred
 * through the UpdateTaskExecutionState message.
 * @param accumulatorSnapshot The serialized flink and user-defined accumulators
 */
public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot) {
	Map<String, Accumulator<?, ?>> userAccumulators;
	try {
		userAccumulators = accumulatorSnapshot.deserializeUserAccumulators(userClassLoader);

		ExecutionAttemptID execID = accumulatorSnapshot.getExecutionAttemptID();
		Execution execution = currentExecutions.get(execID);
		if (execution != null) {
			execution.setAccumulators(userAccumulators);
		} else {
			LOG.debug("Received accumulator result for unknown execution {}.", execID);
		}
	} catch (Exception e) {
		LOG.error("Cannot update accumulators for job {}.", getJobID(), e);
	}
}
 

@Override
public AccumulatorReport retrievePayload(ResourceID resourceID) {
	validateRunsInMainThread();
	JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID);
	if (jobManagerConnection != null) {
		JobID jobId = jobManagerConnection.getJobID();

		List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16);
		Iterator<Task> allTasks = taskSlotTable.getTasks(jobId);

		while (allTasks.hasNext()) {
			Task task = allTasks.next();
			accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot());
		}
		return new AccumulatorReport(accumulatorSnapshots);
	} else {
		return new AccumulatorReport(Collections.emptyList());
	}
}
 

@Override
public void reportPayload(ResourceID resourceID, AccumulatorReport payload) {
	validateRunsInMainThread();
	for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) {
		schedulerNG.updateAccumulators(snapshot);
	}
}
 

private void unregisterTaskAndNotifyFinalState(
		final JobMasterGateway jobMasterGateway,
		final ExecutionAttemptID executionAttemptID) {

	Task task = taskSlotTable.removeTask(executionAttemptID);
	if (task != null) {
		if (!task.getExecutionState().isTerminal()) {
			try {
				task.failExternally(new IllegalStateException("Task is being remove from TaskManager."));
			} catch (Exception e) {
				log.error("Could not properly fail task.", e);
			}
		}

		log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.",
			task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId());

		AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();

		updateTaskExecutionState(
				jobMasterGateway,
				new TaskExecutionState(
					task.getJobID(),
					task.getExecutionId(),
					task.getExecutionState(),
					task.getFailureCause(),
					accumulatorSnapshot,
					task.getMetricGroup().getIOMetricGroup().createSnapshot()));
	} else {
		log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
	}
}
 

private void unregisterTaskAndNotifyFinalState(
		final JobMasterGateway jobMasterGateway,
		final ExecutionAttemptID executionAttemptID) {

	Task task = taskSlotTable.removeTask(executionAttemptID);
	if (task != null) {
		if (!task.getExecutionState().isTerminal()) {
			try {
				task.failExternally(new IllegalStateException("Task is being remove from TaskManager."));
			} catch (Exception e) {
				log.error("Could not properly fail task.", e);
			}
		}

		log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.",
			task.getExecutionState(), task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionId());

		AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();

		updateTaskExecutionState(
				jobMasterGateway,
				new TaskExecutionState(
					task.getJobID(),
					task.getExecutionId(),
					task.getExecutionState(),
					task.getFailureCause(),
					accumulatorSnapshot,
					task.getMetricGroup().getIOMetricGroup().createSnapshot()));
	} else {
		log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
	}
}
 

private void unregisterTaskAndNotifyFinalState(
		final JobMasterGateway jobMasterGateway,
		final ExecutionAttemptID executionAttemptID) {

	Task task = taskSlotTable.removeTask(executionAttemptID);
	if (task != null) {
		if (!task.getExecutionState().isTerminal()) {
			try {
				task.failExternally(new IllegalStateException("Task is being remove from TaskManager."));
			} catch (Exception e) {
				log.error("Could not properly fail task.", e);
			}
		}

		log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.",
			task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId());

		AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();

		updateTaskExecutionState(
				jobMasterGateway,
				new TaskExecutionState(
					task.getJobID(),
					task.getExecutionId(),
					task.getExecutionState(),
					task.getFailureCause(),
					accumulatorSnapshot,
					task.getMetricGroup().getIOMetricGroup().createSnapshot()));
	} else {
		log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
	}
}
 

@Override
public void reportPayload(ResourceID resourceID, AccumulatorReport payload) {
	validateRunsInMainThread();
	for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) {
		executionGraph.updateAccumulators(snapshot);
	}
}
 

@Override
public void reportPayload(ResourceID resourceID, AccumulatorReport payload) {
	validateRunsInMainThread();
	for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) {
		schedulerNG.updateAccumulators(snapshot);
	}
}
 

@Override
public void updateAccumulators(final AccumulatorSnapshot accumulatorSnapshot) {
	mainThreadExecutor.assertRunningInMainThread();

	executionGraph.updateAccumulators(accumulatorSnapshot);
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}
 

public Collection<AccumulatorSnapshot> getAccumulatorSnapshots() {
	return accumulatorSnapshots;
}
 

public AccumulatorReport(List<AccumulatorSnapshot> accumulatorSnapshots) {
	this.accumulatorSnapshots = accumulatorSnapshots;
}
 

/**
 * Gets flink and user-defined accumulators in serialized form.
 */
public AccumulatorSnapshot getAccumulators() {
	return accumulators;
}
 

/**
 * Gets flink and user-defined accumulators in serialized form.
 */
public AccumulatorSnapshot getAccumulators() {
	return accumulators;
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}
 

@Override
public void updateAccumulators(final AccumulatorSnapshot accumulatorSnapshot) {
	mainThreadExecutor.assertRunningInMainThread();

	executionGraph.updateAccumulators(accumulatorSnapshot);
}
 

public Collection<AccumulatorSnapshot> getAccumulatorSnapshots() {
	return accumulatorSnapshots;
}
 

public AccumulatorReport(List<AccumulatorSnapshot> accumulatorSnapshots) {
	this.accumulatorSnapshots = accumulatorSnapshots;
}
 

/**
 * Gets flink and user-defined accumulators in serialized form.
 */
public AccumulatorSnapshot getAccumulators() {
	return accumulators;
}
 

/**
 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
 * execution that failed or was canceled.
 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
	final JobVertexID jid1 = new JobVertexID();
	final JobVertexID jid2 = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", jid1);
	JobVertex v2 = new JobVertex("v2", jid2);

	Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
	ExecutionGraph graph = graphAndExecutions.f0;

	// verify behavior for canceled executions
	Execution execution1 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
	Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
	accumulators.put("acc", new IntCounter(4));
	AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);

	TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);

	graph.updateState(state);

	assertEquals(ioMetrics, execution1.getIOMetrics());
	assertNotNull(execution1.getUserAccumulators());
	assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());

	// verify behavior for failed executions
	Execution execution2 = graphAndExecutions.f1.values().iterator().next();

	IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
	Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
	accumulators2.put("acc", new IntCounter(8));
	AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);

	TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);

	graph.updateState(state2);

	assertEquals(ioMetrics2, execution2.getIOMetrics());
	assertNotNull(execution2.getUserAccumulators());
	assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}