org.apache.flink.runtime.entrypoint.ClusterInformation Java Examples

The following examples show how to use org.apache.flink.runtime.entrypoint.ClusterInformation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ActiveResourceManagerFactoryTest.java From flink with Apache License 2.0

6 votes

@Override
protected ResourceManager<ResourceID> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	assertThat(configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE), is(true));

	return null;
}

Example #2

Source File: TaskExecutorTest.java From flink with Apache License 2.0

6 votes

private TestingResourceManagerGateway createRmWithTmRegisterAndNotifySlotHooks(
		InstanceID registrationId,
		OneShotLatch taskExecutorIsRegistered,
		CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture) {
	final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	resourceManagerLeaderRetriever.notifyListener(
		resourceManagerGateway.getAddress(),
		resourceManagerGateway.getFencingToken().toUUID());

	resourceManagerGateway.setRegisterTaskExecutorFunction(
		taskExecutorRegistration -> CompletableFuture.completedFuture(
			new TaskExecutorRegistrationSuccess(registrationId, resourceManagerGateway.getOwnResourceId(),
				new ClusterInformation("localhost", 1234))));

	resourceManagerGateway.setNotifySlotAvailableConsumer(availableSlotFuture::complete);

	resourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorIsRegistered.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());
	});
	return resourceManagerGateway;
}

Example #3

Source File: TaskExecutor.java From flink with Apache License 2.0

6 votes

@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				try {
					establishResourceManagerConnection(
						resourceManagerGateway,
						resourceManagerId,
						taskExecutorRegistrationId,
						clusterInformation);
				} catch (Throwable t) {
					log.error("Establishing Resource Manager connection in Task Executor failed", t);
				}
			}
		});
}

Example #4

Source File: ActiveResourceManagerFactory.java From flink with Apache License 2.0

6 votes

@Override
public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		MetricRegistry metricRegistry,
		String hostname) throws Exception {
	return super.createResourceManager(
		createActiveResourceManagerConfiguration(configuration),
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		metricRegistry,
		hostname);
}

Example #5

Source File: TaskExecutor.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				establishResourceManagerConnection(
					resourceManagerGateway,
					resourceManagerId,
					taskExecutorRegistrationId,
					clusterInformation);
			}
		});
}

Example #6

Source File: StandaloneResourceManager.java From Flink-CEPplus with Apache License 2.0

6 votes

public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
}

Example #7

Source File: KubernetesResourceManagerTest.java From flink with Apache License 2.0

6 votes

private TestingKubernetesResourceManager createAndStartResourceManager(Configuration configuration, SlotManager slotManager, FlinkKubeClient flinkKubeClient) throws Exception {

			final TestingRpcService rpcService = new TestingRpcService(configuration);
			final MockResourceManagerRuntimeServices rmServices = new MockResourceManagerRuntimeServices(rpcService, TIMEOUT, slotManager);

			final TestingKubernetesResourceManager kubernetesResourceManager = new TestingKubernetesResourceManager(
				rpcService,
				ResourceID.generate(),
				configuration,
				rmServices.highAvailabilityServices,
				rmServices.heartbeatServices,
				rmServices.slotManager,
				rmServices.jobLeaderIdService,
				new ClusterInformation("localhost", 1234),
				testingFatalErrorHandlerResource.getFatalErrorHandler(),
				UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
				flinkKubeClient,
				new KubernetesResourceManagerConfiguration(CLUSTER_ID, TESTING_POD_CREATION_RETRY_INTERVAL));
			kubernetesResourceManager.start();
			rmServices.grantLeadership();
			return kubernetesResourceManager;
		}

Example #8

Source File: TestingResourceManager.java From flink with Apache License 2.0

6 votes

public TestingResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		jobManagerMetricGroup);
}

Example #9

Source File: ActiveResourceManagerFactoryTest.java From flink with Apache License 2.0

6 votes

/**
 * Test which ensures that the {@link ActiveResourceManagerFactory} sets the correct managed
 * memory when creating a resource manager.
 */
@Test
public void createResourceManager_WithDefaultConfiguration_ShouldSetManagedMemory() throws Exception {
	final Configuration configuration = new Configuration();

	final TestingActiveResourceManagerFactory resourceManagerFactory = new TestingActiveResourceManagerFactory();

	final TestingRpcService rpcService = new TestingRpcService();

	try {
		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}

Example #10

Source File: TestingResourceManager.java From Flink-CEPplus with Apache License 2.0

6 votes

public TestingResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		jobManagerMetricGroup);
}

Example #11

Source File: StandaloneResourceManagerFactoryTest.java From flink with Apache License 2.0

6 votes

@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}

Example #12

Source File: TestingResourceManager.java From flink with Apache License 2.0

6 votes

public TestingResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		resourceManagerMetricGroup,
		RpcUtils.INF_TIMEOUT);
}

Example #13

Source File: StandaloneResourceManagerTest.java From flink with Apache License 2.0

6 votes

private TestingStandaloneResourceManager createResourceManager(Time startupPeriod, SlotManager slotManager) throws Exception {

		final MockResourceManagerRuntimeServices rmServices = new MockResourceManagerRuntimeServices(
			RPC_SERVICE.getTestingRpcService(),
			TIMEOUT,
			slotManager);

		final TestingStandaloneResourceManager rm = new TestingStandaloneResourceManager(
			rmServices.rpcService,
			ResourceID.generate(),
			rmServices.highAvailabilityServices,
			rmServices.heartbeatServices,
			rmServices.slotManager,
			rmServices.jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
			startupPeriod,
			rmServices);

		rm.start();
		rmServices.grantLeadership();

		return rm;
	}

Example #14

Source File: YarnResourceManagerTest.java From flink with Apache License 2.0

6 votes

Context(Configuration configuration) throws  Exception {
	rpcService = new TestingRpcService();
	rmServices = new MockResourceManagerRuntimeServices(rpcService, TIMEOUT);

	// resource manager
	rmResourceID = ResourceID.generate();
	resourceManager =
			new TestingYarnResourceManager(
					rpcService,
					RM_ADDRESS,
					rmResourceID,
					configuration,
					env,
					rmServices.highAvailabilityServices,
					rmServices.heartbeatServices,
					rmServices.slotManager,
					rmServices.metricRegistry,
					rmServices.jobLeaderIdService,
					new ClusterInformation("localhost", 1234),
					testingFatalErrorHandler,
					null,
					mockResourceManagerClient,
					mockNMClient,
					mockJMMetricGroup);
}

Example #15

Source File: TaskExecutor.java From flink with Apache License 2.0

6 votes

@Override
public void onRegistrationSuccess(TaskExecutorToResourceManagerConnection connection, TaskExecutorRegistrationSuccess success) {
	final ResourceID resourceManagerId = success.getResourceManagerId();
	final InstanceID taskExecutorRegistrationId = success.getRegistrationId();
	final ClusterInformation clusterInformation = success.getClusterInformation();
	final ResourceManagerGateway resourceManagerGateway = connection.getTargetGateway();

	runAsync(
		() -> {
			// filter out outdated connections
			//noinspection ObjectEquality
			if (resourceManagerConnection == connection) {
				establishResourceManagerConnection(
					resourceManagerGateway,
					resourceManagerId,
					taskExecutorRegistrationId,
					clusterInformation);
			}
		});
}

Example #16

Source File: ActiveResourceManagerFactory.java From flink with Apache License 2.0

6 votes

@Override
public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	return createActiveResourceManager(
		createActiveResourceManagerConfiguration(configuration),
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		jobManagerMetricGroup);
}

Example #17

Source File: ResourceManagerFactory.java From flink with Apache License 2.0

5 votes

ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
JobManagerMetricGroup jobManagerMetricGroup) throws Exception;

Example #18

Source File: KubernetesResourceManager.java From flink with Apache License 2.0

5 votes

public KubernetesResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		Configuration flinkConfig,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		FlinkKubeClient kubeClient,
		KubernetesResourceManagerConfiguration configuration) {
	super(
		flinkConfig,
		System.getenv(),
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup);
	this.clusterId = configuration.getClusterId();
	this.kubeClient = kubeClient;
	this.configuration = configuration;
	this.podWorkerResources = new HashMap<>();
}

Example #19

Source File: YarnResourceManagerTest.java From flink with Apache License 2.0

5 votes

Context(Configuration configuration, @Nullable SlotManager slotManager) throws  Exception {

			workerResourceSpec = YarnWorkerResourceSpecFactory.INSTANCE.createDefaultWorkerResourceSpec(configuration);
			if (slotManager == null) {
				slotManager = SlotManagerBuilder.newBuilder()
					.setDefaultWorkerResourceSpec(workerResourceSpec)
					.build();
			}
			rpcService = new TestingRpcService();
			rmServices = new MockResourceManagerRuntimeServices(rpcService, TIMEOUT, slotManager);

			// resource manager
			rmResourceID = ResourceID.generate();
			resourceManager =
					new TestingYarnResourceManager(
							rpcService,
							rmResourceID,
							configuration,
							env,
							rmServices.highAvailabilityServices,
							rmServices.heartbeatServices,
							rmServices.slotManager,
							rmServices.jobLeaderIdService,
							new ClusterInformation("localhost", 1234),
							testingFatalErrorHandler,
							null,
							UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());

			testingYarnAMRMClientAsync = resourceManager.testingYarnAMRMClientAsync;
			testingYarnNMClientAsync = resourceManager.testingYarnNMClientAsync;

			containerResource = resourceManager.getContainerResource(workerResourceSpec).get();
		}

Example #20

Source File: KubernetesResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<KubernetesWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final KubernetesResourceManagerConfiguration kubernetesResourceManagerConfiguration =
		new KubernetesResourceManagerConfiguration(
			configuration.getString(KubernetesConfigOptions.CLUSTER_ID),
			POD_CREATION_RETRY_INTERVAL);

	return new KubernetesResourceManager(
		rpcService,
		resourceId,
		configuration,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		KubeClientFactory.fromConfiguration(configuration),
		kubernetesResourceManagerConfiguration);
}

Example #21

Source File: TaskExecutorRegistrationSuccess.java From flink with Apache License 2.0

5 votes

/**
 * Create a new {@code TaskExecutorRegistrationSuccess} message.
 *
 * @param registrationId The ID that the ResourceManager assigned the registration.
 * @param resourceManagerResourceId The unique ID that identifies the ResourceManager.
 * @param clusterInformation information about the cluster
 */
public TaskExecutorRegistrationSuccess(
		InstanceID registrationId,
		ResourceID resourceManagerResourceId,
		ClusterInformation clusterInformation) {
	this.registrationId = Preconditions.checkNotNull(registrationId);
	this.resourceManagerResourceId = Preconditions.checkNotNull(resourceManagerResourceId);
	this.clusterInformation = Preconditions.checkNotNull(clusterInformation);
}

Example #22

Source File: StandaloneResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}

Example #23

Source File: TaskExecutorTest.java From flink with Apache License 2.0

5 votes

@Test
public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
	final String resourceManagerAddress = "/resource/manager/address/one";

	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CountDownLatch taskManagerRegisteredLatch = new CountDownLatch(1);
	testingResourceManagerGateway.setRegisterTaskExecutorFunction(FunctionUtils.uncheckedFunction(
		ignored -> {
			taskManagerRegisteredLatch.countDown();
			return CompletableFuture.completedFuture(new TaskExecutorRegistrationSuccess(
				new InstanceID(), new ResourceID(resourceManagerAddress), new ClusterInformation("localhost", 1234)));
		}
	));

	rpc.registerGateway(resourceManagerAddress, testingResourceManagerGateway);

	final TaskSlotTable taskSlotTable = new TaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService);
	final TaskExecutorLocalStateStoresManager localStateStoresManager = createTaskExecutorLocalStateStoresManager();
	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskManagerLocation(taskManagerLocation)
		.setTaskSlotTable(taskSlotTable)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
		resourceManagerLeaderRetriever.notifyListener(resourceManagerAddress, UUID.randomUUID());

		assertTrue(taskManagerRegisteredLatch.await(timeout.toMilliseconds(), TimeUnit.MILLISECONDS));
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}

Example #24

Source File: ActiveResourceManager.java From flink with Apache License 2.0

5 votes

public ActiveResourceManager(
		Configuration flinkConfig,
		Map<String, String> env,
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		AkkaUtils.getTimeoutAsTime(flinkConfig));

	this.flinkConfig = flinkConfig;
	this.env = env;

	// Load the flink config uploaded by flink client
	this.flinkClientConfig = loadClientConfiguration();

	requestedNotAllocatedWorkerCounter = new PendingWorkerCounter();
	requestedNotRegisteredWorkerCounter = new PendingWorkerCounter();
	allocatedNotRegisteredWorkerResourceSpecs = new HashMap<>();
}

Example #25

Source File: TestingResourceManagerGateway.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<RegistrationResponse> registerTaskExecutor(TaskExecutorRegistration taskExecutorRegistration, Time timeout) {
	final Function<TaskExecutorRegistration, CompletableFuture<RegistrationResponse>> currentFunction = registerTaskExecutorFunction;

	if (currentFunction != null) {
		return currentFunction.apply(taskExecutorRegistration);
	} else {
		return CompletableFuture.completedFuture(
			new TaskExecutorRegistrationSuccess(
				new InstanceID(),
				ownResourceId,
				new ClusterInformation("localhost", 1234)));
	}
}

Example #26

Source File: ResourceManagerFactory.java From flink with Apache License 2.0

5 votes

public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		MetricRegistry metricRegistry,
		String hostname) throws Exception {

	final ResourceManagerMetricGroup resourceManagerMetricGroup = ResourceManagerMetricGroup.create(metricRegistry, hostname);
	final SlotManagerMetricGroup slotManagerMetricGroup = SlotManagerMetricGroup.create(metricRegistry, hostname);

	final ResourceManagerRuntimeServices resourceManagerRuntimeServices = createResourceManagerRuntimeServices(
		configuration, rpcService, highAvailabilityServices, slotManagerMetricGroup);

	return createResourceManager(
		configuration,
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		resourceManagerMetricGroup,
		resourceManagerRuntimeServices);
}

Example #27

Source File: ResourceManager.java From flink with Apache License 2.0

5 votes

public ResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {

	super(rpcService, resourceManagerEndpointId);

	this.resourceId = checkNotNull(resourceId);
	this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.slotManager = checkNotNull(slotManager);
	this.metricRegistry = checkNotNull(metricRegistry);
	this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
	this.clusterInformation = checkNotNull(clusterInformation);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);

	this.jobManagerRegistrations = new HashMap<>(4);
	this.jmResourceIdRegistrations = new HashMap<>(4);
	this.taskExecutors = new HashMap<>(8);
	this.taskExecutorGatewayFutures = new HashMap<>(8);
}

Example #28

Source File: StandaloneResourceManager.java From flink with Apache License 2.0

5 votes

public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup,
		Time startupPeriodTime) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
	this.startupPeriodTime = Preconditions.checkNotNull(startupPeriodTime);
}

Example #29

Source File: YarnResourceManagerTest.java From flink with Apache License 2.0

5 votes

TestingYarnResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpResourceManagerPartitionTracker::get,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		resourceManagerMetricGroup);
	this.testingYarnNMClientAsync = new TestingYarnNMClientAsync(this);
	this.testingYarnAMRMClientAsync = new TestingYarnAMRMClientAsync(this);
}

Example #30

Source File: ResourceManagerFactory.java From flink with Apache License 2.0

5 votes

protected abstract ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
ResourceManagerMetricGroup resourceManagerMetricGroup,
ResourceManagerRuntimeServices resourceManagerRuntimeServices) throws Exception;