Java Code Examples for org.apache.flink.runtime.entrypoint.ClusterInformation

The following examples show how to use org.apache.flink.runtime.entrypoint.ClusterInformation. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				establishResourceManagerConnection(
					resourceManagerGateway,
					resourceManagerId,
					taskExecutorRegistrationId,
					clusterInformation);
			}
		});
}
 
Example 2
Source Project: Flink-CEPplus   Source File: StandaloneResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: TestingResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
public TestingResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		jobManagerMetricGroup);
}
 
Example 4
Source Project: flink   Source File: TestingResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
public TestingResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		resourceManagerMetricGroup,
		RpcUtils.INF_TIMEOUT);
}
 
Example 5
Source Project: flink   Source File: StandaloneResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
private TestingStandaloneResourceManager createResourceManager(Time startupPeriod, SlotManager slotManager) throws Exception {

		final MockResourceManagerRuntimeServices rmServices = new MockResourceManagerRuntimeServices(
			RPC_SERVICE.getTestingRpcService(),
			TIMEOUT,
			slotManager);

		final TestingStandaloneResourceManager rm = new TestingStandaloneResourceManager(
			rmServices.rpcService,
			ResourceID.generate(),
			rmServices.highAvailabilityServices,
			rmServices.heartbeatServices,
			rmServices.slotManager,
			rmServices.jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
			startupPeriod,
			rmServices);

		rm.start();
		rmServices.grantLeadership();

		return rm;
	}
 
Example 6
Source Project: flink   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
Context(Configuration configuration) throws  Exception {
	rpcService = new TestingRpcService();
	rmServices = new MockResourceManagerRuntimeServices(rpcService, TIMEOUT);

	// resource manager
	rmResourceID = ResourceID.generate();
	resourceManager =
			new TestingYarnResourceManager(
					rpcService,
					RM_ADDRESS,
					rmResourceID,
					configuration,
					env,
					rmServices.highAvailabilityServices,
					rmServices.heartbeatServices,
					rmServices.slotManager,
					rmServices.metricRegistry,
					rmServices.jobLeaderIdService,
					new ClusterInformation("localhost", 1234),
					testingFatalErrorHandler,
					null,
					mockResourceManagerClient,
					mockNMClient,
					mockJMMetricGroup);
}
 
Example 7
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				establishResourceManagerConnection(
					resourceManagerGateway,
					resourceManagerId,
					taskExecutorRegistrationId,
					clusterInformation);
			}
		});
}
 
Example 8
Source Project: flink   Source File: ActiveResourceManagerFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	return createActiveResourceManager(
		createActiveResourceManagerConfiguration(configuration),
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		jobManagerMetricGroup);
}
 
Example 9
@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 
Example 10
Source Project: flink   Source File: ActiveResourceManagerFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test which ensures that the {@link ActiveResourceManagerFactory} sets the correct managed
 * memory when creating a resource manager.
 */
@Test
public void createResourceManager_WithDefaultConfiguration_ShouldSetManagedMemory() throws Exception {
	final Configuration configuration = new Configuration();

	final TestingActiveResourceManagerFactory resourceManagerFactory = new TestingActiveResourceManagerFactory();

	final TestingRpcService rpcService = new TestingRpcService();

	try {
		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 
Example 11
Source Project: flink   Source File: ActiveResourceManagerFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	assertThat(configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE), is(true));

	return null;
}
 
Example 12
Source Project: flink   Source File: TestingResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
public TestingResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		jobManagerMetricGroup);
}
 
Example 13
Source Project: flink   Source File: KubernetesResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
private TestingKubernetesResourceManager createAndStartResourceManager(Configuration configuration, SlotManager slotManager, FlinkKubeClient flinkKubeClient) throws Exception {

			final TestingRpcService rpcService = new TestingRpcService(configuration);
			final MockResourceManagerRuntimeServices rmServices = new MockResourceManagerRuntimeServices(rpcService, TIMEOUT, slotManager);

			final TestingKubernetesResourceManager kubernetesResourceManager = new TestingKubernetesResourceManager(
				rpcService,
				ResourceID.generate(),
				configuration,
				rmServices.highAvailabilityServices,
				rmServices.heartbeatServices,
				rmServices.slotManager,
				rmServices.jobLeaderIdService,
				new ClusterInformation("localhost", 1234),
				testingFatalErrorHandlerResource.getFatalErrorHandler(),
				UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
				flinkKubeClient,
				new KubernetesResourceManagerConfiguration(CLUSTER_ID, TESTING_POD_CREATION_RETRY_INTERVAL));
			kubernetesResourceManager.start();
			rmServices.grantLeadership();
			return kubernetesResourceManager;
		}
 
Example 14
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				try {
					establishResourceManagerConnection(
						resourceManagerGateway,
						resourceManagerId,
						taskExecutorRegistrationId,
						clusterInformation);
				} catch (Throwable t) {
					log.error("Establishing Resource Manager connection in Task Executor failed", t);
				}
			}
		});
}
 
Example 15
Source Project: flink   Source File: ActiveResourceManagerFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		MetricRegistry metricRegistry,
		String hostname) throws Exception {
	return super.createResourceManager(
		createActiveResourceManagerConfiguration(configuration),
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		metricRegistry,
		hostname);
}
 
Example 16
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
private TestingResourceManagerGateway createRmWithTmRegisterAndNotifySlotHooks(
		InstanceID registrationId,
		OneShotLatch taskExecutorIsRegistered,
		CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture) {
	final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	resourceManagerLeaderRetriever.notifyListener(
		resourceManagerGateway.getAddress(),
		resourceManagerGateway.getFencingToken().toUUID());

	resourceManagerGateway.setRegisterTaskExecutorFunction(
		taskExecutorRegistration -> CompletableFuture.completedFuture(
			new TaskExecutorRegistrationSuccess(registrationId, resourceManagerGateway.getOwnResourceId(),
				new ClusterInformation("localhost", 1234))));

	resourceManagerGateway.setNotifySlotAvailableConsumer(availableSlotFuture::complete);

	resourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorIsRegistered.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());
	});
	return resourceManagerGateway;
}
 
Example 17
Source Project: Flink-CEPplus   Source File: MesosResourceManagerFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ResourceManager<RegisteredMesosWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	final ResourceManagerRuntimeServicesConfiguration rmServicesConfiguration = ResourceManagerRuntimeServicesConfiguration.fromConfiguration(configuration);
	final ResourceManagerRuntimeServices rmRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration(
		rmServicesConfiguration,
		highAvailabilityServices,
		rpcService.getScheduledExecutor());

	return new MesosResourceManager(
		rpcService,
		getEndpointId(),
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		rmRuntimeServices.getSlotManager(),
		metricRegistry,
		rmRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		configuration,
		mesosServices,
		schedulerConfiguration,
		taskManagerParameters,
		taskManagerContainerSpec,
		webInterfaceUrl,
		jobManagerMetricGroup);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: MesosResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
public TestingMesosResourceManager(
	RpcService rpcService,
	String resourceManagerEndpointId,
	ResourceID resourceId,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	SlotManager slotManager,
	MetricRegistry metricRegistry,
	JobLeaderIdService jobLeaderIdService,
	FatalErrorHandler fatalErrorHandler,

	// Mesos specifics
	Configuration flinkConfig,
	MesosServices mesosServices,
	MesosConfiguration mesosConfig,
	MesosTaskManagerParameters taskManagerParameters,
	ContainerSpecification taskManagerContainerSpec,
	JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		flinkConfig,
		mesosServices,
		mesosConfig,
		taskManagerParameters,
		taskManagerContainerSpec,
		null,
		jobManagerMetricGroup);
}
 
Example 19
Source Project: Flink-CEPplus   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
 
Example 20
Source Project: flink   Source File: ResourceManagerTaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpResourceManagerPartitionTracker::get,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
			Time.minutes(5L),
			RpcUtils.INF_TIMEOUT);

	resourceManager.start();

	return resourceManager;
}
 
Example 21
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegisterWithDefaultSlotResourceProfile() throws Exception {
	final int numberOfSlots = 2;
	final TaskExecutor taskExecutor = createTaskExecutor(numberOfSlots);

	taskExecutor.start();

	try {
		final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
		final CompletableFuture<ResourceProfile> registeredDefaultSlotResourceProfileFuture = new CompletableFuture<>();

		final ResourceID ownResourceId = testingResourceManagerGateway.getOwnResourceId();
		testingResourceManagerGateway.setRegisterTaskExecutorFunction(taskExecutorRegistration -> {
			registeredDefaultSlotResourceProfileFuture.complete(taskExecutorRegistration.getDefaultSlotResourceProfile());
			return CompletableFuture.completedFuture(
				new TaskExecutorRegistrationSuccess(
					new InstanceID(),
					ownResourceId,
					new ClusterInformation("localhost", 1234)));
		});

		rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
		resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

		assertThat(registeredDefaultSlotResourceProfileFuture.get(),
			equalTo(TaskExecutorResourceUtils.generateDefaultSlotResourceProfile(TM_RESOURCE_SPEC, numberOfSlots)));
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
public ResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {

	super(rpcService, resourceManagerEndpointId);

	this.resourceId = checkNotNull(resourceId);
	this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.slotManager = checkNotNull(slotManager);
	this.metricRegistry = checkNotNull(metricRegistry);
	this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
	this.clusterInformation = checkNotNull(clusterInformation);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);

	this.jobManagerRegistrations = new HashMap<>(4);
	this.jmResourceIdRegistrations = new HashMap<>(4);
	this.taskExecutors = new HashMap<>(8);
	infoMessageListeners = new ConcurrentHashMap<>(8);
	this.taskExecutorGatewayFutures = new HashMap<>(8);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ResourceManagerFactory.java    License: Apache License 2.0 5 votes vote down vote up
ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
JobManagerMetricGroup jobManagerMetricGroup) throws Exception;
 
Example 24
Source Project: Flink-CEPplus   Source File: TaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
	final String resourceManagerAddress = "/resource/manager/address/one";

	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CountDownLatch taskManagerRegisteredLatch = new CountDownLatch(1);
	testingResourceManagerGateway.setRegisterTaskExecutorFunction(FunctionUtils.uncheckedFunction(
		ignored -> {
			taskManagerRegisteredLatch.countDown();
			return CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(
				new InstanceID(), new ResourceID(resourceManagerAddress), new ClusterInformation("localhost", 1234)));
		}
	));

	rpc.registerGateway(resourceManagerAddress, testingResourceManagerGateway);

	final TaskSlotTable taskSlotTable = new TaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService);
	final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();
	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskManagerLocation(taskManagerLocation)
		.setTaskSlotTable(taskSlotTable)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
		resourceManagerLeaderRetriever.notifyListener(resourceManagerAddress, UUID.randomUUID());

		assertTrue(taskManagerRegisteredLatch.await(timeout.toMilliseconds(), TimeUnit.MILLISECONDS));
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 
Example 25
private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpMetricRegistry.INSTANCE,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 
Example 26
@Override
public CompletableFuture<RegistrationResponse> registerTaskExecutor(String taskExecutorAddress, ResourceID resourceId, int dataPort, HardwareDescription hardwareDescription, Time timeout) {
	final Function<Tuple4<String, ResourceID, Integer, HardwareDescription>, CompletableFuture<RegistrationResponse>> currentFunction = registerTaskExecutorFunction;

	if (currentFunction != null) {
		return currentFunction.apply(Tuple4.of(taskExecutorAddress, resourceId, dataPort, hardwareDescription));
	} else {
		return CompletableFuture.completedFuture(
			new TaskExecutorRegistrationSuccess(
				new InstanceID(),
				ownResourceId,
				new ClusterInformation("localhost", 1234)));
	}
}
 
Example 27
private ResourceManager<?> createAndStartResourceManager() throws Exception {
	ResourceID rmResourceId = ResourceID.generate();

	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, 1000L);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		haServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	ResourceManager<?> resourceManager = new StandaloneResourceManager(
		rpcService,
		ResourceManager.RESOURCE_MANAGER_NAME,
		rmResourceId,
		haServices,
		heartbeatServices,
		slotManager,
		NoOpMetricRegistry.INSTANCE,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 
Example 28
Source Project: flink   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
 
Example 29
Source Project: flink   Source File: TaskExecutorRegistrationSuccess.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new {@code TaskExecutorRegistrationSuccess} message.
 *
 * @param registrationId The ID that the ResourceManager assigned the registration.
 * @param resourceManagerResourceId The unique ID that identifies the ResourceManager.
 * @param clusterInformation information about the cluster
 */
public TaskExecutorRegistrationSuccess(
		InstanceID registrationId,
		ResourceID resourceManagerResourceId,
		ClusterInformation clusterInformation) {
	this.registrationId = Preconditions.checkNotNull(registrationId);
	this.resourceManagerResourceId = Preconditions.checkNotNull(resourceManagerResourceId);
	this.clusterInformation = Preconditions.checkNotNull(clusterInformation);
}
 
Example 30
Source Project: flink   Source File: StandaloneResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup,
		Time startupPeriodTime) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
	this.startupPeriodTime = Preconditions.checkNotNull(startupPeriodTime);
}