Java Code Examples for org.apache.flink.runtime.jobgraph.JobStatus#values()

The following examples show how to use org.apache.flink.runtime.jobgraph.JobStatus#values() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 
Example 2
public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
 
Example 3
Source Project: flink   File: WebMonitorMessagesTest.java    License: Apache License 2.0 6 votes vote down vote up
private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 
Example 4
public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
 
Example 5
/**
 * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
 *
 * @param executionGraph to create the ArchivedExecutionGraph from
 * @return ArchivedExecutionGraph created from the given ExecutionGraph
 */
public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) {
	final int numberVertices = executionGraph.getTotalNumberOfVertices();

	Map<JobVertexID, ArchivedExecutionJobVertex> archivedTasks = new HashMap<>(numberVertices);
	List<ArchivedExecutionJobVertex> archivedVerticesInCreationOrder = new ArrayList<>(numberVertices);

	for (ExecutionJobVertex task : executionGraph.getVerticesTopologically()) {
		ArchivedExecutionJobVertex archivedTask = task.archive();
		archivedVerticesInCreationOrder.add(archivedTask);
		archivedTasks.put(task.getJobVertexId(), archivedTask);
	}

	final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators =
		executionGraph.getAccumulatorsSerialized();

	final long[] timestamps = new long[JobStatus.values().length];

	for (JobStatus jobStatus : JobStatus.values()) {
		final int ordinal = jobStatus.ordinal();
		timestamps[ordinal] = executionGraph.getStatusTimestamp(jobStatus);
	}

	return new ArchivedExecutionGraph(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		archivedTasks,
		archivedVerticesInCreationOrder,
		timestamps,
		executionGraph.getState(),
		executionGraph.getFailureInfo(),
		executionGraph.getJsonPlan(),
		executionGraph.getAccumulatorResultsStringified(),
		serializedUserAccumulators,
		executionGraph.getArchivedExecutionConfig(),
		executionGraph.isStoppable(),
		executionGraph.getCheckpointCoordinatorConfiguration(),
		executionGraph.getCheckpointStatsSnapshot());
}
 
Example 6
@Override
protected JobDetailsInfo getTestResponseInstance() throws Exception {
	final Random random = new Random();
	final int numJobVertexDetailsInfos = 4;
	final String jsonPlan = "{\"id\":\"1234\"}";

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
	final Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(numJobVertexDetailsInfos);
	final Map<ExecutionState, Integer> jobVerticesPerState = new HashMap<>(ExecutionState.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, random.nextLong());
	}

	for (int i = 0; i < numJobVertexDetailsInfos; i++) {
		jobVertexInfos.add(createJobVertexDetailsInfo(random));
	}

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerState.put(executionState, random.nextInt());
	}

	return new JobDetailsInfo(
		new JobID(),
		"foobar",
		true,
		JobStatus.values()[random.nextInt(JobStatus.values().length)],
		1L,
		2L,
		1L,
		1984L,
		timestamps,
		jobVertexInfos,
		jobVerticesPerState,
		jsonPlan);
}
 
Example 7
Source Project: flink   File: ArchivedExecutionGraph.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
 *
 * @param executionGraph to create the ArchivedExecutionGraph from
 * @return ArchivedExecutionGraph created from the given ExecutionGraph
 */
public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) {
	final int numberVertices = executionGraph.getTotalNumberOfVertices();

	Map<JobVertexID, ArchivedExecutionJobVertex> archivedTasks = new HashMap<>(numberVertices);
	List<ArchivedExecutionJobVertex> archivedVerticesInCreationOrder = new ArrayList<>(numberVertices);

	for (ExecutionJobVertex task : executionGraph.getVerticesTopologically()) {
		ArchivedExecutionJobVertex archivedTask = task.archive();
		archivedVerticesInCreationOrder.add(archivedTask);
		archivedTasks.put(task.getJobVertexId(), archivedTask);
	}

	final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators =
		executionGraph.getAccumulatorsSerialized();

	final long[] timestamps = new long[JobStatus.values().length];

	for (JobStatus jobStatus : JobStatus.values()) {
		final int ordinal = jobStatus.ordinal();
		timestamps[ordinal] = executionGraph.getStatusTimestamp(jobStatus);
	}

	return new ArchivedExecutionGraph(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		archivedTasks,
		archivedVerticesInCreationOrder,
		timestamps,
		executionGraph.getState(),
		executionGraph.getFailureInfo(),
		executionGraph.getJsonPlan(),
		executionGraph.getAccumulatorResultsStringified(),
		serializedUserAccumulators,
		executionGraph.getArchivedExecutionConfig(),
		executionGraph.isStoppable(),
		executionGraph.getCheckpointCoordinatorConfiguration(),
		executionGraph.getCheckpointStatsSnapshot());
}
 
Example 8
Source Project: flink   File: JobDetailsInfoTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected JobDetailsInfo getTestResponseInstance() throws Exception {
	final Random random = new Random();
	final int numJobVertexDetailsInfos = 4;
	final String jsonPlan = "{\"id\":\"1234\"}";

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
	final Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(numJobVertexDetailsInfos);
	final Map<ExecutionState, Integer> jobVerticesPerState = new HashMap<>(ExecutionState.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, random.nextLong());
	}

	for (int i = 0; i < numJobVertexDetailsInfos; i++) {
		jobVertexInfos.add(createJobVertexDetailsInfo(random));
	}

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerState.put(executionState, random.nextInt());
	}

	return new JobDetailsInfo(
		new JobID(),
		"foobar",
		true,
		JobStatus.values()[random.nextInt(JobStatus.values().length)],
		1L,
		2L,
		1L,
		1984L,
		timestamps,
		jobVertexInfos,
		jobVerticesPerState,
		jsonPlan);
}
 
Example 9
Source Project: Flink-CEPplus   File: JobDetailsHandler.java    License: Apache License 2.0 4 votes vote down vote up
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher metricFetcher) {
	final long now = System.currentTimeMillis();
	final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED);
	final long endTime = executionGraph.getState().isGloballyTerminalState() ?
		executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L;
	final long duration = (endTime > 0L ? endTime : now) - startTime;

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
	}

	Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size());
	int[] jobVerticesPerState = new int[ExecutionState.values().length];

	for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) {
		final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo(
			accessExecutionJobVertex,
			now,
			executionGraph.getJobID(),
			metricFetcher);

		jobVertexInfos.add(vertexDetailsInfo);
		jobVerticesPerState[vertexDetailsInfo.getExecutionState().ordinal()]++;
	}

	Map<ExecutionState, Integer> jobVerticesPerStateMap = new HashMap<>(ExecutionState.values().length);

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerStateMap.put(executionState, jobVerticesPerState[executionState.ordinal()]);
	}

	return new JobDetailsInfo(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		executionGraph.isStoppable(),
		executionGraph.getState(),
		startTime,
		endTime,
		duration,
		now,
		timestamps,
		jobVertexInfos,
		jobVerticesPerStateMap,
		executionGraph.getJsonPlan());
}
 
Example 10
Source Project: Flink-CEPplus   File: ExecutionGraph.java    License: Apache License 2.0 4 votes vote down vote up
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProvider = Preconditions.checkNotNull(slotProvider, "scheduler");
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();
	this.executionListeners = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 
Example 11
public static JobStatus randomJobStatus(Random rnd) {
	return JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
}
 
Example 12
Source Project: flink   File: JobDetailsHandler.java    License: Apache License 2.0 4 votes vote down vote up
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher metricFetcher) {
	final long now = System.currentTimeMillis();
	final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED);
	final long endTime = executionGraph.getState().isGloballyTerminalState() ?
		executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L;
	final long duration = (endTime > 0L ? endTime : now) - startTime;

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
	}

	Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size());
	int[] jobVerticesPerState = new int[ExecutionState.values().length];

	for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) {
		final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo(
			accessExecutionJobVertex,
			now,
			executionGraph.getJobID(),
			metricFetcher);

		jobVertexInfos.add(vertexDetailsInfo);
		jobVerticesPerState[vertexDetailsInfo.getExecutionState().ordinal()]++;
	}

	Map<ExecutionState, Integer> jobVerticesPerStateMap = new HashMap<>(ExecutionState.values().length);

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerStateMap.put(executionState, jobVerticesPerState[executionState.ordinal()]);
	}

	return new JobDetailsInfo(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		executionGraph.isStoppable(),
		executionGraph.getState(),
		startTime,
		endTime,
		duration,
		now,
		timestamps,
		jobVertexInfos,
		jobVerticesPerStateMap,
		executionGraph.getJsonPlan());
}
 
Example 13
Source Project: flink   File: ExecutionGraph.java    License: Apache License 2.0 4 votes vote down vote up
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker,
		ScheduleMode scheduleMode,
		boolean allowQueuedScheduling) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.allowQueuedScheduling = allowQueuedScheduling;

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout,
		allowQueuedScheduling);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 
Example 14
Source Project: flink   File: GenericMessageTester.java    License: Apache License 2.0 4 votes vote down vote up
public static JobStatus randomJobStatus(Random rnd) {
	return JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
}