org.apache.flink.runtime.resourcemanager.ResourceManagerGateway Java Examples

The following examples show how to use org.apache.flink.runtime.resourcemanager.ResourceManagerGateway. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: apache   File: PartialDispatcherServices.java    License: Apache License 2.0 6 votes vote down vote up
public PartialDispatcherServices(
		@Nonnull Configuration configuration,
		@Nonnull HighAvailabilityServices highAvailabilityServices,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull BlobServer blobServer,
		@Nonnull HeartbeatServices heartbeatServices,
		@Nonnull JobManagerMetricGroupFactory jobManagerMetricGroupFactory,
		@Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore,
		@Nonnull FatalErrorHandler fatalErrorHandler,
		@Nonnull HistoryServerArchivist historyServerArchivist,
		@Nullable String metricQueryServiceAddress) {
	this.configuration = configuration;
	this.highAvailabilityServices = highAvailabilityServices;
	this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever;
	this.blobServer = blobServer;
	this.heartbeatServices = heartbeatServices;
	this.jobManagerMetricGroupFactory = jobManagerMetricGroupFactory;
	this.archivedExecutionGraphStore = archivedExecutionGraphStore;
	this.fatalErrorHandler = fatalErrorHandler;
	this.historyServerArchivist = historyServerArchivist;
	this.metricQueryServiceAddress = metricQueryServiceAddress;
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: DispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 
Example #3
Source Project: flink   Author: apache   File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
	return new RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess>(
		log,
		getRpcService(),
		"ResourceManager",
		ResourceManagerGateway.class,
		getTargetAddress(),
		getTargetLeaderId(),
		jobMasterConfiguration.getRetryingRegistrationConfiguration()) {

		@Override
		protected CompletableFuture<RegistrationResponse> invokeRegistration(
				ResourceManagerGateway gateway, ResourceManagerId fencingToken, long timeoutMillis) {
			Time timeout = Time.milliseconds(timeoutMillis);

			return gateway.registerJobManager(
				jobMasterId,
				jobManagerResourceID,
				jobManagerRpcAddress,
				jobID,
				timeout);
		}
	};
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
	return new RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess>(
		log,
		getRpcService(),
		"ResourceManager",
		ResourceManagerGateway.class,
		getTargetAddress(),
		getTargetLeaderId(),
		jobMasterConfiguration.getRetryingRegistrationConfiguration()) {

		@Override
		protected CompletableFuture<RegistrationResponse> invokeRegistration(
				ResourceManagerGateway gateway, ResourceManagerId fencingToken, long timeoutMillis) {
			Time timeout = Time.milliseconds(timeoutMillis);

			return gateway.registerJobManager(
				jobMasterId,
				jobManagerResourceID,
				jobManagerRpcAddress,
				jobID,
				timeout);
		}
	};
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: MiniDispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public MiniDispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: AbstractTaskManagerFileHandler.java    License: Apache License 2.0 6 votes vote down vote up
protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<ResourceID, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(ResourceID resourceId) throws Exception {
					return loadTaskManagerFile(resourceId);
				}
		});
}
 
Example #7
Source Project: flink   Author: apache   File: MiniDispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public MiniDispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ScheduledExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		ExecutionGraphCache executionGraphCache,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		executionGraphCache,
		fatalErrorHandler);
}
 
Example #8
Source Project: flink   Author: apache   File: AbstractTaskManagerFileHandler.java    License: Apache License 2.0 6 votes vote down vote up
protected AbstractTaskManagerFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, M> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders);

	this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever);

	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);

	this.fileBlobKeys = CacheBuilder
		.newBuilder()
		.expireAfterWrite(cacheEntryDuration.toMilliseconds(), TimeUnit.MILLISECONDS)
		.removalListener(this::removeBlob)
		.build(
			new CacheLoader<Tuple2<ResourceID, String>, CompletableFuture<TransientBlobKey>>() {
				@Override
				public CompletableFuture<TransientBlobKey> load(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws Exception {
					return loadTaskManagerFile(taskManagerIdAndFileName);
				}
		});
}
 
Example #9
Source Project: flink   Author: apache   File: ExecutionGraphNotEnoughResourceTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(
		int numSlots,
		SlotPool slotPool,
		TaskManagerLocation taskManagerLocation) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final String jobManagerAddress = "foobar";
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
	slotPool.connectToResourceManager(resourceManagerGateway);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join();

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join();

	return scheduler;
}
 
Example #10
Source Project: flink   Author: apache   File: AbstractTaskManagerFileHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
 
Example #11
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractTaskManagerFileHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: DispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
	return new RetryingRegistration<ResourceManagerId, ResourceManagerGateway, JobMasterRegistrationSuccess>(
		log,
		getRpcService(),
		"ResourceManager",
		ResourceManagerGateway.class,
		getTargetAddress(),
		getTargetLeaderId(),
		jobMasterConfiguration.getRetryingRegistrationConfiguration()) {

		@Override
		protected CompletableFuture<RegistrationResponse> invokeRegistration(
				ResourceManagerGateway gateway, ResourceManagerId fencingToken, long timeoutMillis) {
			Time timeout = Time.milliseconds(timeoutMillis);

			return gateway.registerJobManager(
				jobMasterId,
				jobManagerResourceID,
				jobManagerRpcAddress,
				jobID,
				timeout);
		}
	};
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				establishResourceManagerConnection(
					resourceManagerGateway,
					resourceManagerId,
					taskExecutorRegistrationId,
					clusterInformation);
			}
		});
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: TaskExecutorToResourceManagerConnection.java    License: Apache License 2.0 6 votes vote down vote up
ResourceManagerRegistration(
		Logger log,
		RpcService rpcService,
		String targetAddress,
		ResourceManagerId resourceManagerId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskExecutorAddress,
		ResourceID resourceID,
		int dataPort,
		HardwareDescription hardwareDescription) {

	super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, resourceManagerId, retryingRegistrationConfiguration);
	this.taskExecutorAddress = checkNotNull(taskExecutorAddress);
	this.resourceID = checkNotNull(resourceID);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
private static void setupSlotPool(
	SlotPoolImpl slotPool,
	ResourceManagerGateway resourceManagerGateway,
	ComponentMainThreadExecutor mainThreadExecutable) throws Exception {
	final String jobManagerAddress = "foobar";

	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutable);

	slotPool.connectToResourceManager(resourceManagerGateway);
}
 
Example #17
Source Project: flink   Author: apache   File: RestEndpointFactory.java    License: Apache License 2.0 5 votes vote down vote up
WebMonitorEndpoint<T> createRestEndpoint(
Configuration configuration,
LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
TransientBlobService transientBlobService,
ScheduledExecutorService executor,
MetricFetcher metricFetcher,
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler) throws Exception;
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: DispatcherFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link Dispatcher} of the given type {@link T}.
 */
T createDispatcher(
	Configuration configuration,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
	BlobServer blobServer,
	HeartbeatServices heartbeatServices,
	JobManagerMetricGroup jobManagerMetricGroup,
	@Nullable String metricQueryServicePath,
	ArchivedExecutionGraphStore archivedExecutionGraphStore,
	FatalErrorHandler fatalErrorHandler,
	HistoryServerArchivist historyServerArchivist) throws Exception;
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: MiniDispatcher.java    License: Apache License 2.0 5 votes vote down vote up
public MiniDispatcher(
		RpcService rpcService,
		String endpointId,
		Configuration configuration,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		JobManagerRunnerFactory jobManagerRunnerFactory,
		FatalErrorHandler fatalErrorHandler,
		HistoryServerArchivist historyServerArchivist,
		JobGraph jobGraph,
		JobClusterEntrypoint.ExecutionMode executionMode) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		new SingleJobSubmittedJobGraphStore(jobGraph),
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServicePath,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler,
		historyServerArchivist);

	this.executionMode = checkNotNull(executionMode);
	this.jobTerminationFuture = new CompletableFuture<>();
}
 
Example #20
Source Project: flink   Author: apache   File: ClusterDataSetDeleteHandlers.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterDataSetDeleteTriggerHandler(
	GatewayRetriever<? extends RestfulGateway> leaderRetriever,
	Time timeout,
	Map<String, String> responseHeaders,
	GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		ClusterDataSetDeleteTriggerHeaders.INSTANCE);
	this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever;
}
 
Example #21
Source Project: flink   Author: apache   File: TaskManagerCustomLogHandler.java    License: Apache License 2.0 5 votes vote down vote up
public TaskManagerCustomLogHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, TaskManagerFileMessageParameters> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders, resourceManagerGatewayRetriever, transientBlobService, cacheEntryDuration);

}
 
Example #22
Source Project: flink   Author: flink-tpc-ds   File: DispatcherResourceManagerComponent.java    License: Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> deregisterApplication(
		final ApplicationStatus applicationStatus,
		final @Nullable String diagnostics) {

	final ResourceManagerGateway selfGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);
	return selfGateway.deregisterApplication(applicationStatus, diagnostics).thenApply(ack -> null);
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void closeResourceManagerConnection(Exception cause) {
	if (establishedResourceManagerConnection != null) {
		final ResourceID resourceManagerResourceId = establishedResourceManagerConnection.getResourceManagerResourceId();

		if (log.isDebugEnabled()) {
			log.debug("Close ResourceManager connection {}.",
				resourceManagerResourceId, cause);
		} else {
			log.info("Close ResourceManager connection {}.",
				resourceManagerResourceId);
		}
		resourceManagerHeartbeatManager.unmonitorTarget(resourceManagerResourceId);

		ResourceManagerGateway resourceManagerGateway = establishedResourceManagerConnection.getResourceManagerGateway();
		resourceManagerGateway.disconnectTaskManager(getResourceID(), cause);

		establishedResourceManagerConnection = null;
	}

	if (resourceManagerConnection != null) {
		if (!resourceManagerConnection.isConnected()) {
			if (log.isDebugEnabled()) {
				log.debug("Terminating registration attempts towards ResourceManager {}.",
					resourceManagerConnection.getTargetAddress(), cause);
			} else {
				log.info("Terminating registration attempts towards ResourceManager {}.",
					resourceManagerConnection.getTargetAddress());
			}
		}

		resourceManagerConnection.close();
		resourceManagerConnection = null;
	}

	startRegistrationTimeout();
}
 
Example #24
Source Project: flink   Author: apache   File: SlotPoolInteractionsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot allocation times out wrt to the specified time out.
 */
@Test
public void testSlotAllocationTimeout() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		pool.connectToResourceManager(resourceManagerGateway);

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());
	}
}
 
Example #25
Source Project: flink   Author: apache   File: TaskManagerStdoutFileHandler.java    License: Apache License 2.0 5 votes vote down vote up
public TaskManagerStdoutFileHandler(
		@Nonnull GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		@Nonnull Time timeout,
		@Nonnull Map<String, String> responseHeaders,
		@Nonnull UntypedResponseMessageHeaders<EmptyRequestBody, TaskManagerMessageParameters> untypedResponseMessageHeaders,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull TransientBlobService transientBlobService,
		@Nonnull Time cacheEntryDuration) {
	super(leaderRetriever, timeout, responseHeaders, untypedResponseMessageHeaders, resourceManagerGatewayRetriever, transientBlobService, cacheEntryDuration);
}
 
Example #26
Source Project: flink   Author: apache   File: SlotPoolImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void connectToResourceManager(@Nonnull ResourceManagerGateway resourceManagerGateway) {
	this.resourceManagerGateway = checkNotNull(resourceManagerGateway);

	// work on all slots waiting for this connection
	for (PendingRequest pendingRequest : waitingForResourceManager.values()) {
		requestSlotFromResourceManager(resourceManagerGateway, pendingRequest);
	}

	// all sent off
	waitingForResourceManager.clear();
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: TaskExecutorToResourceManagerConnection.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected CompletableFuture<RegistrationResponse> invokeRegistration(
		ResourceManagerGateway resourceManager, ResourceManagerId fencingToken, long timeoutMillis) throws Exception {

	Time timeout = Time.milliseconds(timeoutMillis);
	return resourceManager.registerTaskExecutor(
		taskExecutorAddress,
		resourceID,
		dataPort,
		hardwareDescription,
		timeout);
}
 
Example #28
Source Project: Flink-CEPplus   Author: ljygz   File: RestEndpointFactory.java    License: Apache License 2.0 5 votes vote down vote up
WebMonitorEndpoint<T> createRestEndpoint(
Configuration configuration,
LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
TransientBlobService transientBlobService,
ExecutorService executor,
MetricFetcher metricFetcher,
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler) throws Exception;
 
Example #29
Source Project: flink   Author: apache   File: AbstractTaskManagerFileHandler.java    License: Apache License 2.0 5 votes vote down vote up
private CompletableFuture<TransientBlobKey> loadTaskManagerFile(Tuple2<ResourceID, String> taskManagerIdAndFileName) throws RestHandlerException {
	log.debug("Load file from TaskManager {}.", taskManagerIdAndFileName.f0);

	final ResourceManagerGateway resourceManagerGateway = resourceManagerGatewayRetriever
		.getNow()
		.orElseThrow(() -> {
			log.debug("Could not connect to ResourceManager right now.");
			return new RestHandlerException(
				"Cannot connect to ResourceManager right now. Please try to refresh.",
				HttpResponseStatus.NOT_FOUND);
		});

	return requestFileUpload(resourceManagerGateway, taskManagerIdAndFileName);
}
 
Example #30
Source Project: flink   Author: apache   File: JobMaster.java    License: Apache License 2.0 5 votes vote down vote up
private void dissolveResourceManagerConnection(EstablishedResourceManagerConnection establishedResourceManagerConnection, Exception cause) {
	final ResourceID resourceManagerResourceID = establishedResourceManagerConnection.getResourceManagerResourceID();

	if (log.isDebugEnabled()) {
		log.debug("Close ResourceManager connection {}.", resourceManagerResourceID, cause);
	} else {
		log.info("Close ResourceManager connection {}: {}.", resourceManagerResourceID, cause.getMessage());
	}

	resourceManagerHeartbeatManager.unmonitorTarget(resourceManagerResourceID);

	ResourceManagerGateway resourceManagerGateway = establishedResourceManagerConnection.getResourceManagerGateway();
	resourceManagerGateway.disconnectJobManager(jobGraph.getJobID(), cause);
	slotPool.disconnectResourceManager();
}