org.apache.flink.api.common.time.Time Java Examples

@Override
public <V> CompletableFuture<V> callAsyncWithoutFencing(Callable<V> callable, Time timeout) {
	checkNotNull(callable, "callable");
	checkNotNull(timeout, "timeout");

	if (isLocal) {
		@SuppressWarnings("unchecked")
		CompletableFuture<V> resultFuture = (CompletableFuture<V>) FutureUtils.toJava(
			Patterns.ask(
				getActorRef(),
				new UnfencedMessage<>(new CallAsync(callable)),
				timeout.toMilliseconds()));

		return resultFuture;
	} else {
		throw new RuntimeException("Trying to send a Runnable to a remote actor at " +
			getActorRef().path() + ". This is not supported.");
	}
}
 

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 

public SlotPoolImpl(
		JobID jobId,
		Clock clock,
		Time rpcTimeout,
		Time idleSlotTimeout) {

	this.jobId = checkNotNull(jobId);
	this.clock = checkNotNull(clock);
	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.idleSlotTimeout = checkNotNull(idleSlotTimeout);

	this.registeredTaskManagers = new HashSet<>(16);
	this.allocatedSlots = new AllocatedSlots();
	this.availableSlots = new AvailableSlots();
	this.pendingRequests = new DualKeyMap<>(16);
	this.waitingForResourceManager = new HashMap<>(16);

	this.jobMasterId = null;
	this.resourceManagerGateway = null;
	this.jobManagerAddress = null;

	this.componentMainThreadExecutor = null;
}
 

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws Exception {
					return loadTaskManagerFile(taskManagerIdAndFileName);
				}
		});
}
 

public JarPlanHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<JarPlanRequestBody, JobPlanInfo, JarPlanMessageParameters> messageHeaders,
		final Path jarDir,
		final Configuration configuration,
		final Executor executor) {
	this(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		jarDir,
		configuration,
		executor,
		jobGraph -> new JobPlanInfo(JsonPlanGenerator.generatePlan(jobGraph)));
}
 

/**
 * Tests dynamically loading of handlers such as {@link JarUploadHandler}.
 */
@Test
public void testLoadWebSubmissionExtension() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.ADDRESS, "localhost");
	final WebMonitorExtension webMonitorExtension = WebMonitorUtils.loadWebSubmissionExtension(
		CompletableFuture::new,
		Time.seconds(10),
		Collections.emptyMap(),
		CompletableFuture.completedFuture("localhost:12345"),
		Paths.get("/tmp"),
		Executors.directExecutor(),
		configuration);

	assertThat(webMonitorExtension, is(not(nullValue())));
}
 

@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		try {
			task.cancelExecution();
			return CompletableFuture.completedFuture(Acknowledge.get());
		} catch (Throwable t) {
			return FutureUtils.completedExceptionally(
				new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t));
		}
	} else {
		final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new TaskException(message));
	}
}
 

/**
 * Convenience constructor for tests. Sets various fields to default values.
 */
@VisibleForTesting
ExecutionVertex(
		ExecutionJobVertex jobVertex,
		int subTaskIndex,
		IntermediateResult[] producedDataSets,
		Time timeout) {

	this(
		jobVertex,
		subTaskIndex,
		producedDataSets,
		timeout,
		1L,
		System.currentTimeMillis(),
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue());
}
 

@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 

@Override
public CompletableFuture<JobStatus> requestJobStatus(JobID jobId, Time timeout) {

	final CompletableFuture<JobMasterGateway> jobMasterGatewayFuture = getJobMasterGatewayFuture(jobId);

	final CompletableFuture<JobStatus> jobStatusFuture = jobMasterGatewayFuture.thenCompose(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	return jobStatusFuture.exceptionally(
		(Throwable throwable) -> {
			final JobDetails jobDetails = archivedExecutionGraphStore.getAvailableJobDetails(jobId);

			// check whether it is a completed job
			if (jobDetails == null) {
				throw new CompletionException(ExceptionUtils.stripCompletionException(throwable));
			} else {
				return jobDetails.getStatus();
			}
		});
}
 

@Override
public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) {
	CompletableFuture<ResourceOverview> taskManagerOverviewFuture = runResourceManagerCommand(resourceManagerGateway -> resourceManagerGateway.requestResourceOverview(timeout));

	final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation);

	CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection);

	final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview();

	return allJobsFuture.thenCombine(
		taskManagerOverviewFuture,
		(Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> {
			final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview);
			return new ClusterOverview(resourceOverview, allJobsOverview);
		});
}
 

AkkaInvocationHandler(
		String address,
		String hostname,
		ActorRef rpcEndpoint,
		Time timeout,
		long maximumFramesize,
		@Nullable CompletableFuture<Void> terminationFuture) {

	this.address = Preconditions.checkNotNull(address);
	this.hostname = Preconditions.checkNotNull(hostname);
	this.rpcEndpoint = Preconditions.checkNotNull(rpcEndpoint);
	this.isLocal = this.rpcEndpoint.path().address().hasLocalScope();
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maximumFramesize = maximumFramesize;
	this.terminationFuture = terminationFuture;
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the submit task
 * message fails.
 */
@Test(timeout = 10000L)
public void testSubmitTaskFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor(
		"test task",
		eid,
		BlockingNoOpInvokable.class,
		0); // this will make the submission fail because the number of key groups must be >= 1

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
	} catch (Exception e) {
		assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
	}
}
 

/**
 * Tests that idle task managers time out after the configured timeout. A timed out task manager
 * will be removed from the slot manager and the resource manager will be notified about the
 * timeout, if it can be released.
 */
@Test
public void testTaskManagerTimeout() throws Exception {
	Executor executor = TestingUtils.defaultExecutor();
	canBeReleasedFuture.set(CompletableFuture.completedFuture(true));
	try (SlotManager slotManager = SlotManagerBuilder
		.newBuilder()
		.setTaskManagerTimeout(Time.milliseconds(10L))
		.build()) {

		slotManager.start(resourceManagerId, executor, resourceManagerActions);
		executor.execute(() -> slotManager.registerTaskManager(taskManagerConnection, slotReport));
		assertThat(releaseFuture.get(), is(equalTo(taskManagerConnection.getInstanceID())));
	}
}
 

TestingExecution(
		Executor executor,
		ExecutionVertex vertex,
		int attemptNumber,
		long globalModVersion,
		long startTimestamp,
		Time rpcTimeout,
		ExecutionState state,
		CompletionType completionType,
		long requestTimeout) {
	super(executor, vertex, attemptNumber, globalModVersion, startTimestamp, rpcTimeout);
	this.state = checkNotNull(state);
	this.completionType = checkNotNull(completionType);
	this.requestTimeout = requestTimeout;
}
 

/**
 * Schedule the operation with the given delay.
 *
 * @param operation to schedule
 * @param delay delay to schedule
 * @param scheduledExecutor executor to be used for the operation
 * @param <T> type of the result
 * @return Future which schedules the given operation with given delay.
 */
public static <T> CompletableFuture<T> scheduleWithDelay(
		final Supplier<T> operation,
		final Time delay,
		final ScheduledExecutor scheduledExecutor) {
	final CompletableFuture<T> resultFuture = new CompletableFuture<>();

	ScheduledFuture<?> scheduledFuture = scheduledExecutor.schedule(
		() -> {
			try {
				resultFuture.complete(operation.get());
			} catch (Throwable t) {
				resultFuture.completeExceptionally(t);
			}
		},
		delay.getSize(),
		delay.getUnit()
	);

	resultFuture.whenComplete(
		(t, throwable) -> {
			if (!scheduledFuture.isDone()) {
				scheduledFuture.cancel(false);
			}
		});
	return resultFuture;
}
 

/**
 * Polls the {@link JobStatus} of a job periodically and when the job has reached
 * a terminal state, it requests its {@link JobResult}.
 *
 * @param dispatcherGateway the {@link DispatcherGateway} to be used for requesting the details of the job.
 * @param jobId the id of the job
 * @param scheduledExecutor the executor to be used to periodically request the status of the job
 * @param rpcTimeout the timeout of the rpc
 * @param retryPeriod the interval between two consecutive job status requests
 * @return a future that will contain the job's {@link JobResult}.
 */
static CompletableFuture<JobResult> getJobResult(
		final DispatcherGateway dispatcherGateway,
		final JobID jobId,
		final ScheduledExecutor scheduledExecutor,
		final Time rpcTimeout,
		final Time retryPeriod) {

	return pollJobResultAsync(
			() -> dispatcherGateway.requestJobStatus(jobId, rpcTimeout),
			() -> dispatcherGateway.requestJobResult(jobId, rpcTimeout),
			scheduledExecutor,
			retryPeriod.toMilliseconds()
	);
}
 

@Test
public void testStartupPeriod() throws Exception {
	final LinkedBlockingQueue<Boolean> setFailUnfulfillableRequestInvokes = new LinkedBlockingQueue<>();
	final SlotManager slotManager = new TestingSlotManagerBuilder()
		.setSetFailUnfulfillableRequestConsumer(setFailUnfulfillableRequestInvokes::add)
		.createSlotManager();
	final TestingStandaloneResourceManager rm = createResourceManager(Time.milliseconds(1L), slotManager);

	assertThat(setFailUnfulfillableRequestInvokes.take(), is(false));
	assertThat(setFailUnfulfillableRequestInvokes.take(), is(true));

	rm.close();
}
 

private ExecutionJobVertex buildExecutionJobVertex(OperatorID operatorID, OperatorID userDefinedOperatorId, int parallelism) throws JobException, JobExecutionException {
	ExecutionGraph graph = TestingExecutionGraphBuilder.newBuilder().build();
	JobVertex jobVertex = new JobVertex(
		operatorID.toHexString(),
		new JobVertexID(),
		singletonList(OperatorIDPair.of(operatorID, userDefinedOperatorId)));
	return new ExecutionJobVertex(graph, jobVertex, parallelism, 1, Time.seconds(1), 1L, 1L);
}
 

private static ExecutionVertex mockExecutionVertex(
	ExecutionAttemptID attemptID,
	JobVertexID jobVertexID,
	List<OperatorID> jobVertexIDs,
	int parallelism,
	int maxParallelism,
	ExecutionState state,
	ExecutionState ... successiveStates) {

	ExecutionVertex vertex = mock(ExecutionVertex.class);

	final Execution exec = spy(new Execution(
		mock(Executor.class),
		vertex,
		1,
		1L,
		1L,
		Time.milliseconds(500L)
	));
	when(exec.getAttemptId()).thenReturn(attemptID);
	when(exec.getState()).thenReturn(state, successiveStates);

	when(vertex.getJobvertexId()).thenReturn(jobVertexID);
	when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
	when(vertex.getTotalNumberOfParallelSubtasks()).thenReturn(parallelism);
	when(vertex.getMaxParallelism()).thenReturn(maxParallelism);

	ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
	when(jobVertex.getOperatorIDs()).thenReturn(jobVertexIDs);
	
	when(vertex.getJobVertex()).thenReturn(jobVertex);

	return vertex;
}
 

public static void main(String[] args) {
    long l = System.currentTimeMillis();
    //timestamp - (timestamp - offset + slide) % slide;
    System.out.println(l - (l  + 60 * 1000) %  60000);

    long size = Time.hours(24).toMilliseconds();
    long slide = Time.hours(1).toMilliseconds();
    long lastStart = (1572794063000l - (1572794063000l + slide) % slide);
    for (long start = lastStart; start > 1572794063000l - size; start -= slide) {
        System.out.println(start + "  " + (start + size));
    }
}
 

@Override
public CompletableFuture<ResourceOverview> requestResourceOverview(Time timeout) {
	final int numberSlots = slotManager.getNumberRegisteredSlots();
	final int numberFreeSlots = slotManager.getNumberFreeSlots();

	return CompletableFuture.completedFuture(
		new ResourceOverview(
			taskExecutors.size(),
			numberSlots,
			numberFreeSlots));
}
 

private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	final Time timeout = Time.days(1L);
	ExecutionGraph executionGraph = new ExecutionGraph(
		new DummyJobInformation(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		new NoRestartStrategy(),
		new RestartAllStrategy.Factory(),
		new TestingSlotProvider(slotRequestId -> CompletableFuture.completedFuture(new TestingLogicalSlot())),
		ClassLoader.getSystemClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	executionGraph.enableCheckpointing(
			100,
			100,
			100,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	executionGraph.attachJobGraph(Collections.singletonList(jobVertex));
	executionGraph.setQueuedSchedulingAllowed(true);

	return executionGraph;
}
 

@Test
public void testShouldThrowExceptionIfTaskIsNotRunningBeforeSampling() {
	try {
		stackTraceSampleService.requestStackTraceSample(
			new NotRunningTask(),
			10,
			Time.milliseconds(0),
			-1);
		fail("Expected exception not thrown");
	} catch (final IllegalStateException e) {
		assertThat(e.getMessage(), containsString("Cannot sample task"));
	}
}
 

/**
 * Retry the given operation with the given delay in between failures.
 *
 * @param operation to retry
 * @param retries number of retries
 * @param retryDelay delay between retries
 * @param scheduledExecutor executor to be used for the retry operation
 * @param <T> type of the result
 * @return Future which retries the given operation a given amount of times and delays the retry in case of failures
 */
public static <T> CompletableFuture<T> retryWithDelay(
		final Supplier<CompletableFuture<T>> operation,
		final int retries,
		final Time retryDelay,
		final ScheduledExecutor scheduledExecutor) {
	return retryWithDelay(
		operation,
		retries,
		retryDelay,
		(throwable) -> true,
		scheduledExecutor);
}
 

public DefaultSlotPoolFactory(
		@Nonnull Clock clock,
		@Nonnull Time rpcTimeout,
		@Nonnull Time slotIdleTimeout,
		@Nonnull Time batchSlotTimeout) {
	this.clock = clock;
	this.rpcTimeout = rpcTimeout;
	this.slotIdleTimeout = slotIdleTimeout;
	this.batchSlotTimeout = batchSlotTimeout;
}
 

public RpcResultPartitionConsumableNotifier(
		JobMasterGateway jobMasterGateway,
		Executor executor,
		Time timeout) {
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.executor = Preconditions.checkNotNull(executor);
	this.timeout = Preconditions.checkNotNull(timeout);
}
 

@Override
public <V> CompletableFuture<V> callAsync(Callable<V> callable, Time callTimeout) {
	if (isLocal) {
		@SuppressWarnings("unchecked")
		CompletableFuture<V> resultFuture = (CompletableFuture<V>) ask(new CallAsync(callable), callTimeout);

		return resultFuture;
	} else {
		throw new RuntimeException("Trying to send a Callable to a remote actor at " +
			rpcEndpoint.path() + ". This is not supported.");
	}
}
 

public ShutdownHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> messageHeaders) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders);
}
 

public JobManagerMetricsHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> headers,
		final MetricFetcher metricFetcher) {
	super(leaderRetriever, timeout, headers, JobManagerMetricsHeaders.getInstance(), metricFetcher);
}