org.apache.flink.api.common.time.Time Java Examples

@Override
public <V> CompletableFuture<V> callAsyncWithoutFencing(Callable<V> callable, Time timeout) {
	checkNotNull(callable, "callable");
	checkNotNull(timeout, "timeout");

	if (isLocal) {
		@SuppressWarnings("unchecked")
		CompletableFuture<V> resultFuture = (CompletableFuture<V>) FutureUtils.toJava(
			Patterns.ask(
				getActorRef(),
				new UnfencedMessage<>(new CallAsync(callable)),
				timeout.toMilliseconds()));

		return resultFuture;
	} else {
		throw new RuntimeException("Trying to send a Runnable to a remote actor at " +
			getActorRef().path() + ". This is not supported.");
	}
}
 

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 

@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the submit task
 * message fails.
 */
@Test(timeout = 10000L)
public void testSubmitTaskFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor(
		"test task",
		eid,
		BlockingNoOpInvokable.class,
		0); // this will make the submission fail because the number of key groups must be >= 1

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
	} catch (Exception e) {
		assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
	}
}
 

public SlotPoolImpl(
		JobID jobId,
		Clock clock,
		Time rpcTimeout,
		Time idleSlotTimeout) {

	this.jobId = checkNotNull(jobId);
	this.clock = checkNotNull(clock);
	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.idleSlotTimeout = checkNotNull(idleSlotTimeout);

	this.registeredTaskManagers = new HashSet<>(16);
	this.allocatedSlots = new AllocatedSlots();
	this.availableSlots = new AvailableSlots();
	this.pendingRequests = new DualKeyMap<>(16);
	this.waitingForResourceManager = new HashMap<>(16);

	this.jobMasterId = null;
	this.resourceManagerGateway = null;
	this.jobManagerAddress = null;

	this.componentMainThreadExecutor = null;
}
 

/**
 * Convenience constructor for tests. Sets various fields to default values.
 */
@VisibleForTesting
ExecutionVertex(
		ExecutionJobVertex jobVertex,
		int subTaskIndex,
		IntermediateResult[] producedDataSets,
		Time timeout) {

	this(
		jobVertex,
		subTaskIndex,
		producedDataSets,
		timeout,
		1L,
		System.currentTimeMillis(),
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue());
}
 

@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		try {
			task.cancelExecution();
			return CompletableFuture.completedFuture(Acknowledge.get());
		} catch (Throwable t) {
			return FutureUtils.completedExceptionally(
				new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t));
		}
	} else {
		final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new TaskException(message));
	}
}
 

/**
 * Tests dynamically loading of handlers such as {@link JarUploadHandler}.
 */
@Test
public void testLoadWebSubmissionExtension() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.ADDRESS, "localhost");
	final WebMonitorExtension webMonitorExtension = WebMonitorUtils.loadWebSubmissionExtension(
		CompletableFuture::new,
		Time.seconds(10),
		Collections.emptyMap(),
		CompletableFuture.completedFuture("localhost:12345"),
		Paths.get("/tmp"),
		Executors.directExecutor(),
		configuration);

	assertThat(webMonitorExtension, is(not(nullValue())));
}
 

AkkaInvocationHandler(
		String address,
		String hostname,
		ActorRef rpcEndpoint,
		Time timeout,
		long maximumFramesize,
		@Nullable CompletableFuture<Void> terminationFuture) {

	this.address = Preconditions.checkNotNull(address);
	this.hostname = Preconditions.checkNotNull(hostname);
	this.rpcEndpoint = Preconditions.checkNotNull(rpcEndpoint);
	this.isLocal = this.rpcEndpoint.path().address().hasLocalScope();
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maximumFramesize = maximumFramesize;
	this.terminationFuture = terminationFuture;
}
 

@Override
public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) {
	CompletableFuture<ResourceOverview> taskManagerOverviewFuture = runResourceManagerCommand(resourceManagerGateway -> resourceManagerGateway.requestResourceOverview(timeout));

	final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation);

	CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection);

	final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview();

	return allJobsFuture.thenCombine(
		taskManagerOverviewFuture,
		(Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> {
			final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview);
			return new ClusterOverview(resourceOverview, allJobsOverview);
		});
}
 

public JarPlanHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<JarPlanRequestBody, JobPlanInfo, JarPlanMessageParameters> messageHeaders,
		final Path jarDir,
		final Configuration configuration,
		final Executor executor) {
	this(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		jarDir,
		configuration,
		executor,
		jobGraph -> new JobPlanInfo(JsonPlanGenerator.generatePlan(jobGraph)));
}
 

@Override
public CompletableFuture<JobStatus> requestJobStatus(JobID jobId, Time timeout) {

	final CompletableFuture<JobMasterGateway> jobMasterGatewayFuture = getJobMasterGatewayFuture(jobId);

	final CompletableFuture<JobStatus> jobStatusFuture = jobMasterGatewayFuture.thenCompose(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	return jobStatusFuture.exceptionally(
		(Throwable throwable) -> {
			final JobDetails jobDetails = archivedExecutionGraphStore.getAvailableJobDetails(jobId);

			// check whether it is a completed job
			if (jobDetails == null) {
				throw new CompletionException(ExceptionUtils.stripCompletionException(throwable));
			} else {
				return jobDetails.getStatus();
			}
		});
}
 

protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws Exception {
					return loadTaskManagerFile(taskManagerIdAndFileName);
				}
		});
}
 

/**
 * Shuts the given rpc services down and waits for their termination.
 *
 * @param rpcServices to shut down
 * @param timeout for this operation
 * @throws InterruptedException if the operation has been interrupted
 * @throws ExecutionException if a problem occurred
 * @throws TimeoutException if a timeout occurred
 */
public static void terminateRpcServices(
		Time timeout,
		RpcService... rpcServices) throws InterruptedException, ExecutionException, TimeoutException {
	terminateAsyncCloseables(
		Arrays.stream(rpcServices)
			.map(rpcService -> (AutoCloseableAsync) rpcService::stopService)
			.collect(Collectors.toList()),
		timeout);
}
 

@Nonnull
@Override
public CompletableFuture<PhysicalSlot> requestNewAllocatedSlot(
		@Nonnull SlotRequestId slotRequestId,
		@Nonnull ResourceProfile resourceProfile,
		Time timeout) {

	componentMainThreadExecutor.assertRunningInMainThread();

	final PendingRequest pendingRequest = PendingRequest.createStreamingRequest(slotRequestId, resourceProfile);

	// register request timeout
	FutureUtils
		.orTimeout(
			pendingRequest.getAllocatedSlotFuture(),
			timeout.toMilliseconds(),
			TimeUnit.MILLISECONDS,
			componentMainThreadExecutor)
		.whenComplete(
			(AllocatedSlot ignored, Throwable throwable) -> {
				if (throwable instanceof TimeoutException) {
					timeoutPendingSlotRequest(slotRequestId);
				}
			});

	return requestNewAllocatedSlotInternal(pendingRequest)
		.thenApply((Function.identity()));
}
 

/**
 * Tests that a task manager timeout does not remove the slots from the SlotManager.
 * A timeout should only trigger the {@link ResourceActions#releaseResource(InstanceID, Exception)}
 * callback. The receiver of the callback can then decide what to do with the TaskManager.
 *
 * <p>See FLINK-7793
 */
@Test
public void testTaskManagerTimeoutDoesNotRemoveSlots() throws Exception {
	final Time taskManagerTimeout = Time.milliseconds(10L);
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceID resourceID = ResourceID.generate();
	final CompletableFuture<InstanceID> releaseResourceFuture = new CompletableFuture<>();
	final ResourceActions resourceActions = new TestingResourceActionsBuilder()
		.setReleaseResourceConsumer((instanceId, ignored) -> releaseResourceFuture.complete(instanceId))
		.build();
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();

	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
	final SlotStatus slotStatus = createEmptySlotStatus(new SlotID(resourceID, 0), ResourceProfile.fromResources(1.0, 1));
	final SlotReport initialSlotReport = new SlotReport(slotStatus);

	try (final SlotManager slotManager = createSlotManagerBuilder()
		.setTaskManagerTimeout(taskManagerTimeout)
		.buildAndStartWithDirectExec(resourceManagerId, resourceActions)) {

		slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport);

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		// wait for the timeout call to happen
		assertThat(releaseResourceFuture.get(), is(taskExecutorConnection.getInstanceID()));

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		slotManager.unregisterTaskManager(taskExecutorConnection.getInstanceID(), TEST_EXCEPTION);

		assertEquals(0, slotManager.getNumberRegisteredSlots());
	}
}
 

static void init() throws Exception {
	jarDir = TMP.newFolder().toPath();

	// properties are set property by surefire plugin
	final String parameterProgramJarName = System.getProperty("parameterJarName") + ".jar";
	final String parameterProgramWithoutManifestJarName = System.getProperty("parameterJarWithoutManifestName") + ".jar";
	final Path jarLocation = Paths.get(System.getProperty("targetDir"));

	jarWithManifest = Files.copy(
		jarLocation.resolve(parameterProgramJarName),
		jarDir.resolve("program-with-manifest.jar"));
	jarWithoutManifest = Files.copy(
		jarLocation.resolve(parameterProgramWithoutManifestJarName),
		jarDir.resolve("program-without-manifest.jar"));

	restfulGateway = new TestingDispatcherGateway.Builder()
		.setBlobServerPort(BLOB_SERVER_RESOURCE.getBlobServerPort())
		.setSubmitFunction(jobGraph -> {
			LAST_SUBMITTED_JOB_GRAPH_REFERENCE.set(jobGraph);
			return CompletableFuture.completedFuture(Acknowledge.get());
		})
		.build();

	gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
	localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345");
	timeout = Time.seconds(10);
	responseHeaders = Collections.emptyMap();
	executor = TestingUtils.defaultExecutor();
}
 

@Override
public CompletableFuture<Acknowledge> setNewFencingToken(UUID fencingToken, Time timeout) {
	enteringSetNewFencingToken.trigger();
	try {
		triggerSetNewFencingToken.await();
	} catch (InterruptedException e) {
		throw new RuntimeException("TriggerSetNewFencingToken OneShotLatch was interrupted.");
	}

	setFencingToken(fencingToken);

	return CompletableFuture.completedFuture(Acknowledge.get());
}
 

@Test
public void testShouldThrowExceptionIfNumSamplesIsNegative() {
	try {
		stackTraceSampleService.requestStackTraceSample(
			new TestTask(),
			-1,
			Time.milliseconds(0),
			10);
		fail("Expected exception not thrown");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(), is(equalTo("numSamples must be positive")));
	}
}
 

@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}
 

public MetricFetcherImpl(
		GatewayRetriever<T> retriever,
		MetricQueryServiceRetriever queryServiceRetriever,
		Executor executor,
		Time timeout,
		long updateInterval) {
	this.retriever = Preconditions.checkNotNull(retriever);
	this.queryServiceRetriever = Preconditions.checkNotNull(queryServiceRetriever);
	this.executor = Preconditions.checkNotNull(executor);
	this.timeout = Preconditions.checkNotNull(timeout);

	Preconditions.checkArgument(updateInterval > 0, "The update interval must be larger than 0.");
	this.updateInterval = updateInterval;
}
 

@Override
public void close() throws Exception {
    while (pendingTransactions.get() > 0) {
        LOG.info("sleeping {}s by pending transactions", pendingTransactions.get());
        Thread.sleep(Time.seconds(pendingTransactions.get()).toMilliseconds());
    }

    if (session == null) return;
    session.close();

    if (client == null) return;
    client.close();
}
 

@Override
public CompletableFuture<TransientBlobKey> requestFileUploadByName(String fileName, Time timeout) {
	final String filePath;
	final String logDir = taskManagerConfiguration.getTaskManagerLogDir();
	if (StringUtils.isNullOrWhitespaceOnly(logDir) || StringUtils.isNullOrWhitespaceOnly(fileName)) {
		filePath = null;
	} else {
		filePath = new File(logDir, new File(fileName).getName()).getPath();
	}
	return requestFileUploadByFilePath(filePath, fileName);
}
 

public JobManagerMetricsHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> headers,
		final MetricFetcher metricFetcher) {
	super(leaderRetriever, timeout, headers, JobManagerMetricsHeaders.getInstance(), metricFetcher);
}
 

public ShutdownHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> messageHeaders) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders);
}
 

@Override
public <V> CompletableFuture<V> callAsync(Callable<V> callable, Time callTimeout) {
	if (isLocal) {
		@SuppressWarnings("unchecked")
		CompletableFuture<V> resultFuture = (CompletableFuture<V>) ask(new CallAsync(callable), callTimeout);

		return resultFuture;
	} else {
		throw new RuntimeException("Trying to send a Callable to a remote actor at " +
			rpcEndpoint.path() + ". This is not supported.");
	}
}
 

@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	index++;

	if (index >= successfulOperations) {
		return FutureUtils.completedExceptionally(new IOException("Rpc call fails"));
	} else {
		return CompletableFuture.completedFuture(Acknowledge.get());
	}
}
 

private File runFileCachingTest(
		Time cacheEntryDuration,
		Time delayBetweenRequests) throws Exception {
	final Queue<CompletableFuture<TransientBlobKey>> requestFileUploads = new ArrayDeque<>(2);
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey1));
	requestFileUploads.add(CompletableFuture.completedFuture(transientBlobKey2));

	final TestTaskManagerFileHandler testTaskManagerFileHandler = createTestTaskManagerFileHandler(
		cacheEntryDuration,
		requestFileUploads,
		EXPECTED_TASK_MANAGER_ID);

	final File outputFile = temporaryFolder.newFile();
	final TestingChannelHandlerContext testingContext = new TestingChannelHandlerContext(outputFile);

	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);

	Thread.sleep(delayBetweenRequests.toMilliseconds());

	// the handler should not trigger the file upload again because it is still cached
	testTaskManagerFileHandler.respondToRequest(
		testingContext,
		HTTP_REQUEST,
		handlerRequest,
		null);
	return outputFile;
}
 

/**
 * Retry the given operation with the given delay in between failures.
 *
 * @param operation to retry
 * @param retries number of retries
 * @param retryDelay delay between retries
 * @param scheduledExecutor executor to be used for the retry operation
 * @param <T> type of the result
 * @return Future which retries the given operation a given amount of times and delays the retry in case of failures
 */
public static <T> CompletableFuture<T> retryWithDelay(
		final Supplier<CompletableFuture<T>> operation,
		final int retries,
		final Time retryDelay,
		final ScheduledExecutor scheduledExecutor) {
	return retryWithDelay(
		operation,
		retries,
		retryDelay,
		(throwable) -> true,
		scheduledExecutor);
}
 

/**
 * Creates a new Execution attempt.
 *
 * @param executor
 *             The executor used to dispatch callbacks from futures and asynchronous RPC calls.
 * @param vertex
 *             The execution vertex to which this Execution belongs
 * @param attemptNumber
 *             The execution attempt number.
 * @param globalModVersion
 *             The global modification version of the execution graph when this execution was created
 * @param startTimestamp
 *             The timestamp that marks the creation of this Execution
 * @param rpcTimeout
 *             The rpcTimeout for RPC calls like deploy/cancel/stop.
 */
public Execution(
		Executor executor,
		ExecutionVertex vertex,
		int attemptNumber,
		long globalModVersion,
		long startTimestamp,
		Time rpcTimeout) {

	this.executor = checkNotNull(executor);
	this.vertex = checkNotNull(vertex);
	this.attemptId = new ExecutionAttemptID();
	this.rpcTimeout = checkNotNull(rpcTimeout);

	this.globalModVersion = globalModVersion;
	this.attemptNumber = attemptNumber;

	this.stateTimestamps = new long[ExecutionState.values().length];
	markTimestamp(CREATED, startTimestamp);

	this.partitionInfos = new ArrayList<>(16);
	this.producedPartitions = Collections.emptyMap();
	this.terminalStateFuture = new CompletableFuture<>();
	this.releaseFuture = new CompletableFuture<>();
	this.taskManagerLocationFuture = new CompletableFuture<>();

	this.assignedResource = null;
}