Java Code Examples for org.apache.flink.runtime.messages.Acknowledge

The following examples show how to use org.apache.flink.runtime.messages.Acknowledge. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: CliFrontendSavepointTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDisposeSavepointSuccess() throws Exception {
	replaceStdOutAndStdErr();

	String savepointPath = "expectedSavepointPath";

	ClusterClient clusterClient = new DisposeSavepointClusterClient(
		(String path) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration());

	try {

		CliFrontend frontend = new MockedCliFrontend(clusterClient);

		String[] parameters = { "-d", savepointPath };
		frontend.savepoint(parameters);

		String outMsg = buffer.toString();
		assertTrue(outMsg.contains(savepointPath));
		assertTrue(outMsg.contains("disposed"));
	}
	finally {
		clusterClient.shutdown();
		restoreStdOutAndStdErr();
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 
Example 3
Source Project: Flink-CEPplus   Source File: CliFrontend.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sends a {@link JobManagerMessages.DisposeSavepoint} message to the job manager.
 */
private void disposeSavepoint(ClusterClient<?> clusterClient, String savepointPath) throws FlinkException {
	Preconditions.checkNotNull(savepointPath, "Missing required argument: savepoint path. " +
		"Usage: bin/flink savepoint -d <savepoint-path>");

	logAndSysout("Disposing savepoint '" + savepointPath + "'.");

	final CompletableFuture<Acknowledge> disposeFuture = clusterClient.disposeSavepoint(savepointPath);

	logAndSysout("Waiting for response...");

	try {
		disposeFuture.get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS);
	} catch (Exception e) {
		throw new FlinkException("Disposing the savepoint '" + savepointPath + "' failed.", e);
	}

	logAndSysout("Savepoint '" + savepointPath + "' disposed.");
}
 
Example 4
Source Project: flink   Source File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> notifyKvStateRegistered(
		final JobID jobId,
		final JobVertexID jobVertexId,
		final KeyGroupRange keyGroupRange,
		final String registrationName,
		final KvStateID kvStateId,
		final InetSocketAddress kvStateServerAddress) {

	try {
		schedulerNG.notifyKvStateRegistered(jobId, jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress);
		return CompletableFuture.completedFuture(Acknowledge.get());
	} catch (FlinkJobNotFoundException e) {
		log.info("Error while receiving notification about key-value state registration", e);
		return FutureUtils.completedExceptionally(e);
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: TestingDispatcherGateway.java    License: Apache License 2.0 5 votes vote down vote up
public TestingDispatcherGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServicePathsSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServicePathsSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		Function<JobGraph, CompletableFuture<Acknowledge>> submitFunction,
		Supplier<CompletableFuture<Collection<JobID>>> listFunction,
		int blobServerPort,
		DispatcherId fencingToken,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestArchivedJobFunction) {
	super(
		address,
		hostname,
		cancelJobFunction,
		stopJobFunction,
		requestJobFunction,
		requestJobResultFunction,
		requestJobStatusFunction,
		requestMultipleJobDetailsSupplier,
		requestClusterOverviewSupplier,
		requestMetricQueryServicePathsSupplier,
		requestTaskManagerMetricQueryServicePathsSupplier,
		requestOperatorBackPressureStatsFunction,
		triggerSavepointFunction);
	this.submitFunction = submitFunction;
	this.listFunction = listFunction;
	this.blobServerPort = blobServerPort;
	this.fencingToken = fencingToken;
	this.requestArchivedJobFunction = requestArchivedJobFunction;
}
 
Example 6
Source Project: flink   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> requestSlot(
		JobMasterId jobMasterId,
		SlotRequest slotRequest,
		final Time timeout) {

	JobID jobId = slotRequest.getJobId();
	JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);

	if (null != jobManagerRegistration) {
		if (Objects.equals(jobMasterId, jobManagerRegistration.getJobMasterId())) {
			log.info("Request slot with profile {} for job {} with allocation id {}.",
				slotRequest.getResourceProfile(),
				slotRequest.getJobId(),
				slotRequest.getAllocationId());

			try {
				slotManager.registerSlotRequest(slotRequest);
			} catch (ResourceManagerException e) {
				return FutureUtils.completedExceptionally(e);
			}

			return CompletableFuture.completedFuture(Acknowledge.get());
		} else {
			return FutureUtils.completedExceptionally(new ResourceManagerException("The job leader's id " +
				jobManagerRegistration.getJobMasterId() + " does not match the received id " + jobMasterId + '.'));
		}

	} else {
		return FutureUtils.completedExceptionally(new ResourceManagerException("Could not find registered job manager for job " + jobId + '.'));
	}
}
 
Example 7
Source Project: flink   Source File: AsyncCallsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> setNewFencingToken(UUID fencingToken, Time timeout) {
	enteringSetNewFencingToken.trigger();
	try {
		triggerSetNewFencingToken.await();
	} catch (InterruptedException e) {
		throw new RuntimeException("TriggerSetNewFencingToken OneShotLatch was interrupted.");
	}

	setFencingToken(fencingToken);

	return CompletableFuture.completedFuture(Acknowledge.get());
}
 
Example 8
Source Project: Flink-CEPplus   Source File: Dispatcher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
	log.info("Received JobGraph submission {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	try {
		if (isDuplicateJob(jobGraph.getJobID())) {
			return FutureUtils.completedExceptionally(
				new JobSubmissionException(jobGraph.getJobID(), "Job has already been submitted."));
		} else {
			return internalSubmitJob(jobGraph);
		}
	} catch (FlinkException e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: JobManagerRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the {@link JobManagerRunner} always waits for the previous leadership operation
 * (granting or revoking leadership) to finish before starting a new leadership operation.
 */
@Test
public void testConcurrentLeadershipOperationsBlockingSuspend() throws Exception {
	final CompletableFuture<Acknowledge> suspendedFuture = new CompletableFuture<>();

	TestingJobMasterServiceFactory jobMasterServiceFactory = new TestingJobMasterServiceFactory(
		() -> new TestingJobMasterService(
			"localhost",
			e -> suspendedFuture));
	JobManagerRunner jobManagerRunner = createJobManagerRunner(jobMasterServiceFactory);

	jobManagerRunner.start();

	leaderElectionService.isLeader(UUID.randomUUID()).get();

	leaderElectionService.notLeader();

	final CompletableFuture<UUID> leaderFuture = leaderElectionService.isLeader(UUID.randomUUID());

	// the new leadership should wait first for the suspension to happen
	assertThat(leaderFuture.isDone(), is(false));

	try {
		leaderFuture.get(1L, TimeUnit.MILLISECONDS);
		fail("Granted leadership even though the JobMaster has not been suspended.");
	} catch (TimeoutException expected) {
		// expected
	}

	suspendedFuture.complete(Acknowledge.get());

	leaderFuture.get();
}
 
Example 10
Source Project: flink   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRejectionOnCountMismatch() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}
	final Path countExceedingFile = TEMPORARY_FOLDER.newFile().toPath();

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(jobGraphFile.toFile(), countExceedingFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		ExceptionUtils.findThrowable(e, candidate -> candidate instanceof RestHandlerException && candidate.getMessage().contains("count"));
	}
}
 
Example 11
/**
 * Tests that the uploaded blobs are being cleaned up in case of a job submission failure.
 */
@Test
public void testBlobServerCleanupWhenJobSubmissionFails() throws Exception {
	failJobMasterCreationWith.set(() -> new FlinkException("Test exception."));
	final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

	try {
		submissionFuture.get();
		fail("Job submission was expected to fail.");
	} catch (ExecutionException ee) {
		assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true));
	}

	assertThatHABlobsHaveBeenRemoved();
}
 
Example 12
Source Project: flink   Source File: SimpleAckingTaskManagerGateway.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> freeSlot(AllocationID allocationId, Throwable cause, Time timeout) {
	final BiFunction<AllocationID, Throwable, CompletableFuture<Acknowledge>> currentFreeSlotFunction = freeSlotFunction;

	if (currentFreeSlotFunction != null) {
		return currentFreeSlotFunction.apply(allocationId, cause);
	} else {
		return CompletableFuture.completedFuture(Acknowledge.get());
	}
}
 
Example 13
Source Project: flink   Source File: SlotManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Rejects the pending slot request by failing the request future with a
 * {@link SlotAllocationException}.
 *
 * @param pendingSlotRequest to reject
 * @param cause of the rejection
 */
private void rejectPendingSlotRequest(PendingSlotRequest pendingSlotRequest, Exception cause) {
	CompletableFuture<Acknowledge> request = pendingSlotRequest.getRequestFuture();

	if (null != request) {
		request.completeExceptionally(new SlotAllocationException(cause));
	} else {
		LOG.debug("Cannot reject pending slot request {}, since no request has been sent.", pendingSlotRequest.getAllocationId());
	}
}
 
Example 14
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we continue reconnecting to the latest known RM after a disconnection
 * message.
 */
@Test
public void testReconnectionAfterDisconnect() throws Exception {
	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
		final TestingResourceManagerGateway testingResourceManagerGateway = createAndRegisterTestingResourceManagerGateway();
		final BlockingQueue<JobMasterId> registrationsQueue = new ArrayBlockingQueue<>(1);

		testingResourceManagerGateway.setRegisterJobManagerConsumer(
			jobMasterIdResourceIDStringJobIDTuple4 -> registrationsQueue.offer(jobMasterIdResourceIDStringJobIDTuple4.f0));

		final ResourceManagerId resourceManagerId = testingResourceManagerGateway.getFencingToken();
		notifyResourceManagerLeaderListeners(testingResourceManagerGateway);

		// wait for first registration attempt
		final JobMasterId firstRegistrationAttempt = registrationsQueue.take();

		assertThat(firstRegistrationAttempt, equalTo(jobMasterId));

		assertThat(registrationsQueue.isEmpty(), is(true));
		jobMasterGateway.disconnectResourceManager(resourceManagerId, new FlinkException("Test exception"));

		// wait for the second registration attempt after the disconnect call
		assertThat(registrationsQueue.take(), equalTo(jobMasterId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: JobMaster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Updates the task execution state for a given task.
 *
 * @param taskExecutionState New task execution state for a given task
 * @return Acknowledge the task execution state update
 */
@Override
public CompletableFuture<Acknowledge> updateTaskExecutionState(
		final TaskExecutionState taskExecutionState) {
	checkNotNull(taskExecutionState, "taskExecutionState");

	if (executionGraph.updateState(taskExecutionState)) {
		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		return FutureUtils.completedExceptionally(
			new ExecutionGraphException("The execution attempt " +
				taskExecutionState.getID() + " was not found."));
	}
}
 
Example 16
Source Project: flink   Source File: CliFrontendSavepointTest.java    License: Apache License 2.0 5 votes vote down vote up
DisposeSavepointClusterClient(
	Function<String, CompletableFuture<Acknowledge>> disposeSavepointFunction,
	Configuration configuration) throws Exception {
	super(configuration, StandaloneClusterId.getInstance());

	this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction);
}
 
Example 17
Source Project: flink   Source File: FencedRpcEndpointTest.java    License: Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Acknowledge> setFencingTokenInMainThread(UUID fencingToken, Time timeout) {
	return callAsyncWithoutFencing(
		() -> {
			setFencingToken(fencingToken);

			return Acknowledge.get();
		},
		timeout);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: TestingRestfulGateway.java    License: Apache License 2.0 5 votes vote down vote up
public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServicePathsSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServicePathsSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.stopJobFunction = stopJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServicePathsSupplier = requestMetricQueryServicePathsSupplier;
	this.requestTaskManagerMetricQueryServicePathsSupplier = requestTaskManagerMetricQueryServicePathsSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
}
 
Example 19
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRequestKvStateWithoutRegistration() throws Exception {
	final JobGraph graph = createKvJobGraph();

	final JobMaster jobMaster = createJobMaster(
		configuration,
		graph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build(),
		heartbeatServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);
	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		// lookup location
		try {
			jobMasterGateway.requestKvStateLocation(graph.getJobID(), "unknown").get();
			fail("Expected to fail with UnknownKvStateLocation");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowable(e, UnknownKvStateLocation.class).isPresent());
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}
 
Example 20
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRequestKvStateWithIrrelevantRegistration() throws Exception {
	final JobGraph graph = createKvJobGraph();

	final JobMaster jobMaster = createJobMaster(
		configuration,
		graph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build(),
		heartbeatServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);
	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		// register an irrelevant KvState
		try {
			jobMasterGateway.notifyKvStateRegistered(
				new JobID(),
				new JobVertexID(),
				new KeyGroupRange(0, 0),
				"any-name",
				new KvStateID(),
				new InetSocketAddress(InetAddress.getLocalHost(), 1233)).get();
			fail("Expected to fail with FlinkJobNotFoundException.");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent());
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}
 
Example 21
Source Project: Flink-CEPplus   Source File: ExecutionGraphStopTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the stopping RPC call is sent upon stopping requests.
 */
@Test
public void testStopRpc() throws Exception {
	final JobID jid = new JobID();
	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(5);

	final ExecutionGraph graph = ExecutionGraphTestUtils.createSimpleTestGraph(jid, vertex);
	final Execution exec = graph.getJobVertex(vertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	final TaskManagerGateway gateway = mock(TaskManagerGateway.class);
	when(gateway.submitTask(any(TaskDeploymentDescriptor.class), any(Time.class)))
			.thenReturn(CompletableFuture.completedFuture(Acknowledge.get()));
	when(gateway.stopTask(any(ExecutionAttemptID.class), any(Time.class)))
			.thenReturn(CompletableFuture.completedFuture(Acknowledge.get()));

	final SimpleSlot slot = ExecutionGraphTestUtils.createMockSimpleSlot(gateway);

	exec.tryAssignResource(slot);
	exec.deploy();
	exec.switchToRunning();
	assertEquals(ExecutionState.RUNNING, exec.getState());

	exec.stop();
	assertEquals(ExecutionState.RUNNING, exec.getState());

	verify(gateway, times(1)).stopTask(any(ExecutionAttemptID.class), any(Time.class));

	exec.markFinished();
	assertEquals(ExecutionState.FINISHED, exec.getState());
}
 
Example 22
Source Project: Flink-CEPplus   Source File: RestClusterClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> rescaleJob(JobID jobId, int newParallelism) {

	final RescalingTriggerHeaders rescalingTriggerHeaders = RescalingTriggerHeaders.getInstance();
	final RescalingTriggerMessageParameters rescalingTriggerMessageParameters = rescalingTriggerHeaders.getUnresolvedMessageParameters();
	rescalingTriggerMessageParameters.jobPathParameter.resolve(jobId);
	rescalingTriggerMessageParameters.rescalingParallelismQueryParameter.resolve(Collections.singletonList(newParallelism));

	final CompletableFuture<TriggerResponse> rescalingTriggerResponseFuture = sendRequest(
		rescalingTriggerHeaders,
		rescalingTriggerMessageParameters);

	final CompletableFuture<AsynchronousOperationInfo> rescalingOperationFuture = rescalingTriggerResponseFuture.thenCompose(
		(TriggerResponse triggerResponse) -> {
			final TriggerId triggerId = triggerResponse.getTriggerId();
			final RescalingStatusHeaders rescalingStatusHeaders = RescalingStatusHeaders.getInstance();
			final RescalingStatusMessageParameters rescalingStatusMessageParameters = rescalingStatusHeaders.getUnresolvedMessageParameters();

			rescalingStatusMessageParameters.jobPathParameter.resolve(jobId);
			rescalingStatusMessageParameters.triggerIdPathParameter.resolve(triggerId);

			return pollResourceAsync(
				() -> sendRequest(
					rescalingStatusHeaders,
					rescalingStatusMessageParameters));
		});

	return rescalingOperationFuture.thenApply(
		(AsynchronousOperationInfo asynchronousOperationInfo) -> {
			if (asynchronousOperationInfo.getFailureCause() == null) {
				return Acknowledge.get();
			} else {
				throw new CompletionException(asynchronousOperationInfo.getFailureCause());
			}
		});
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Cleanup application and shut down cluster.
 *
 * @param finalStatus of the Flink application
 * @param diagnostics diagnostics message for the Flink application or {@code null}
 */
@Override
public CompletableFuture<Acknowledge> deregisterApplication(
		final ApplicationStatus finalStatus,
		@Nullable final String diagnostics) {
	log.info("Shut down cluster because application is in {}, diagnostics {}.", finalStatus, diagnostics);

	try {
		internalDeregisterApplication(finalStatus, diagnostics);
	} catch (ResourceManagerException e) {
		log.warn("Could not properly shutdown the application.", e);
	}

	return CompletableFuture.completedFuture(Acknowledge.get());
}
 
Example 24
Source Project: flink   Source File: TestingJobMasterGatewayBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public TestingJobMasterGatewayBuilder setScheduleOrUpdateConsumersFunction(Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction) {
	this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction;
	return this;
}
 
Example 25
Source Project: flink   Source File: TestingTaskExecutorGatewayBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public TestingTaskExecutorGatewayBuilder setSubmitTaskConsumer(BiFunction<TaskDeploymentDescriptor, JobMasterId, CompletableFuture<Acknowledge>> submitTaskConsumer) {
	this.submitTaskConsumer = submitTaskConsumer;
	return this;
}
 
Example 26
Source Project: flink   Source File: TestingJobMasterGateway.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> cancel(Time timeout) {
	return cancelFunction.get();
}
 
Example 27
Source Project: Flink-CEPplus   Source File: ZooKeeperHADispatcherTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the {@link Dispatcher} releases a locked {@link SubmittedJobGraph} if it
 * lost the leadership.
 */
@Test
public void testSubmittedJobGraphRelease() throws Exception {
	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final CuratorFramework otherClient = ZooKeeperUtils.startCuratorFramework(configuration);

	try (final TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices()) {
		testingHighAvailabilityServices.setSubmittedJobGraphStore(ZooKeeperUtils.createSubmittedJobGraphs(client, configuration));

		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = ZooKeeperUtils.createSubmittedJobGraphs(
			otherClient,
			configuration);

		otherSubmittedJobGraphStore.start(NoOpSubmittedJobGraphListener.INSTANCE);

		final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
		testingHighAvailabilityServices.setDispatcherLeaderElectionService(leaderElectionService);

		final TestingDispatcher dispatcher = createDispatcher(
			testingHighAvailabilityServices,
			new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)));

		dispatcher.start();

		try {
			final DispatcherId expectedLeaderId = DispatcherId.generate();
			leaderElectionService.isLeader(expectedLeaderId.toUUID()).get();

			final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

			final JobGraph nonEmptyJobGraph = DispatcherHATest.createNonEmptyJobGraph();
			final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(nonEmptyJobGraph, TIMEOUT);
			submissionFuture.get();

			Collection<JobID> jobIds = otherSubmittedJobGraphStore.getJobIds();

			final JobID jobId = nonEmptyJobGraph.getJobID();
			assertThat(jobIds, Matchers.contains(jobId));

			leaderElectionService.notLeader();

			// wait for the job to properly terminate
			final CompletableFuture<Void> jobTerminationFuture = dispatcher.getJobTerminationFuture(jobId, TIMEOUT);
			jobTerminationFuture.get();

			// recover the job
			final SubmittedJobGraph submittedJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobId);

			assertThat(submittedJobGraph, is(notNullValue()));

			// check that the other submitted job graph store can remove the job graph after the original leader
			// has lost its leadership
			otherSubmittedJobGraphStore.removeJobGraph(jobId);

			jobIds = otherSubmittedJobGraphStore.getJobIds();

			assertThat(jobIds, Matchers.not(Matchers.contains(jobId)));
		} finally {
			RpcUtils.terminateRpcEndpoint(dispatcher, TIMEOUT);
			client.close();
			otherClient.close();
		}
	}
}
 
Example 28
Source Project: flink   Source File: RpcTaskManagerGateway.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	return taskExecutorGateway.cancelTask(executionAttemptID, timeout);
}
 
Example 29
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPartitionReleaseOnJobTermination() throws Exception {
	final JobManagerSharedServices jobManagerSharedServices = new TestingJobManagerSharedServicesBuilder().build();
	final JobGraph jobGraph = createSingleVertexJobGraph();

	final CompletableFuture<ResourceID> partitionCleanupTaskExecutorId = new CompletableFuture<>();
	final TestingPartitionTracker partitionTracker = new TestingPartitionTracker();
	partitionTracker.setStopTrackingAndReleaseAllPartitionsConsumer(partitionCleanupTaskExecutorId::complete);

	final JobMaster jobMaster = new JobMasterBuilder()
		.withConfiguration(configuration)
		.withJobGraph(jobGraph)
		.withHighAvailabilityServices(haServices)
		.withJobManagerSharedServices(jobManagerSharedServices)
		.withHeartbeatServices(heartbeatServices)
		.withOnCompletionActions(new TestingOnCompletionActions())
		.withPartitionTrackerFactory(ignord -> partitionTracker)
		.createJobMaster();

	final CompletableFuture<TaskDeploymentDescriptor> taskDeploymentDescriptorFuture = new CompletableFuture<>();
	final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsFuture = new CompletableFuture<>();
	final CompletableFuture<JobID> disconnectTaskExecutorFuture = new CompletableFuture<>();
	final TestingTaskExecutorGateway testingTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder()
		.setReleasePartitionsConsumer((jobId, partitions) -> releasePartitionsFuture.complete(Tuple2.of(jobId, partitions)))
		.setDisconnectJobManagerConsumer((jobID, throwable) -> disconnectTaskExecutorFuture.complete(jobID))
		.setSubmitTaskConsumer((tdd, ignored) -> {
			taskDeploymentDescriptorFuture.complete(tdd);
			return CompletableFuture.completedFuture(Acknowledge.get());
		})
		.createTestingTaskExecutorGateway();

	try {
		jobMaster.start(jobMasterId).get();

		final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

		final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		registerSlotsAtJobMaster(1, jobMasterGateway, testingTaskExecutorGateway, taskManagerLocation);

		// update the execution state of the only execution to FINISHED
		// this should trigger the job to finish
		final TaskDeploymentDescriptor taskDeploymentDescriptor = taskDeploymentDescriptorFuture.get();
		jobMasterGateway.updateTaskExecutionState(
			new TaskExecutionState(
				taskDeploymentDescriptor.getJobId(),
				taskDeploymentDescriptor.getExecutionAttemptId(),
				ExecutionState.FINISHED));

		assertThat(partitionCleanupTaskExecutorId.get(), equalTo(taskManagerLocation.getResourceID()));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}
 
Example 30
Source Project: flink   Source File: RestfulGateway.java    License: Apache License 2.0 4 votes vote down vote up
default CompletableFuture<Acknowledge> shutDownCluster() {
	throw new UnsupportedOperationException();
}