org.apache.flink.runtime.highavailability.HighAvailabilityServices Java Examples

The following examples show how to use org.apache.flink.runtime.highavailability.HighAvailabilityServices. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private JobMaster createJobMaster(
		Configuration configuration,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityServices,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices) throws Exception {

	return createJobMaster(
		configuration,
		jobGraph,
		highAvailabilityServices,
		jobManagerSharedServices,
		heartbeatServices,
		new JobMasterBuilder.TestingOnCompletionActions());
}
 
Example #2
Source File: StandaloneHaServicesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example #3
Source File: TestingResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestingResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		resourceManagerMetricGroup,
		RpcUtils.INF_TIMEOUT);
}
 
Example #4
Source File: DefaultJobMasterServiceFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
public DefaultJobMasterServiceFactory(
		JobMasterConfiguration jobMasterConfiguration,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		RpcService rpcService,
		HighAvailabilityServices haServices,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory,
		FatalErrorHandler fatalErrorHandler,
		SchedulerNGFactory schedulerNGFactory,
		ShuffleMaster<?> shuffleMaster) {
	this.jobMasterConfiguration = jobMasterConfiguration;
	this.slotPoolFactory = slotPoolFactory;
	this.schedulerFactory = schedulerFactory;
	this.rpcService = rpcService;
	this.haServices = haServices;
	this.jobManagerSharedServices = jobManagerSharedServices;
	this.heartbeatServices = heartbeatServices;
	this.jobManagerJobMetricGroupFactory = jobManagerJobMetricGroupFactory;
	this.fatalErrorHandler = fatalErrorHandler;
	this.schedulerNGFactory = schedulerNGFactory;
	this.shuffleMaster = shuffleMaster;
}
 
Example #5
Source File: ZooKeeperHADispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private TestingDispatcher createDispatcher(HighAvailabilityServices highAvailabilityServices, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	return new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + '_' + name.getMethodName() + UUID.randomUUID(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		jobManagerRunnerFactory,
		testingFatalErrorHandler);
}
 
Example #6
Source File: ActiveResourceManagerFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	assertThat(configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE), is(true));

	return null;
}
 
Example #7
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreatingTaskManagerRpcServiceShouldFailIfRpcPortRangeIsInvalid() throws Exception {
	final Configuration config = new Configuration(createFlinkConfigWithPredefinedTaskManagerHostname("example.org"));
	config.setString(TaskManagerOptions.RPC_PORT, "-1");

	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	try {
		TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		fail("Should fail because -1 is not a valid port range");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(),  containsString("Invalid port range definition: -1"));
	} finally {
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example #8
Source File: StandaloneHaServicesTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example #9
Source File: ActiveResourceManagerFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public ResourceManager<T> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	return createActiveResourceManager(
		createActiveResourceManagerConfiguration(configuration),
		resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		fatalErrorHandler,
		clusterInformation,
		webInterfaceUrl,
		jobManagerMetricGroup);
}
 
Example #10
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private JobMaster createJobMaster(
		Configuration configuration,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityServices,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		OnCompletionActions onCompletionActions) throws Exception {

	return new JobMasterBuilder()
		.withConfiguration(configuration)
		.withJobGraph(jobGraph)
		.withHighAvailabilityServices(highAvailabilityServices)
		.withJobManagerSharedServices(jobManagerSharedServices)
		.withHeartbeatServices(heartbeatServices)
		.withOnCompletionActions(onCompletionActions)
		.createJobMaster();
}
 
Example #11
Source File: TestingTaskExecutor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TestingTaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler) {
	super(
		rpcService,
		taskManagerConfiguration,
		haServices,
		taskExecutorServices,
		heartbeatServices,
		taskManagerMetricGroup,
		metricQueryServicePath,
		blobCacheService,
		fatalErrorHandler);
}
 
Example #12
Source File: StandaloneHaServicesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election services return a fixed address and leader session
 * id.
 */
@Test
public void testLeaderElection() throws Exception {
	JobID jobId = new JobID();
	LeaderContender jmLeaderContender = mock(LeaderContender.class);
	LeaderContender rmLeaderContender = mock(LeaderContender.class);

	LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();

	jmLeaderElectionService.start(jmLeaderContender);
	rmLeaderElectionService.start(rmLeaderContender);

	verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example #13
Source File: PartialDispatcherServicesWithJobGraphStore.java    From flink with Apache License 2.0 6 votes vote down vote up
public PartialDispatcherServicesWithJobGraphStore(
		@Nonnull Configuration configuration,
		@Nonnull HighAvailabilityServices highAvailabilityServices,
		@Nonnull GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		@Nonnull BlobServer blobServer,
		@Nonnull HeartbeatServices heartbeatServices,
		@Nonnull JobManagerMetricGroupFactory jobManagerMetricGroupFactory,
		@Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore,
		@Nonnull FatalErrorHandler fatalErrorHandler,
		@Nonnull HistoryServerArchivist historyServerArchivist,
		@Nullable String metricQueryServiceAddress,
		@Nonnull JobGraphWriter jobGraphWriter) {
	super(
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroupFactory,
		archivedExecutionGraphStore,
		fatalErrorHandler,
		historyServerArchivist,
		metricQueryServiceAddress);
	this.jobGraphWriter = jobGraphWriter;
}
 
Example #14
Source File: ResourceManagerRuntimeServices.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ResourceManagerRuntimeServices fromConfiguration(
		ResourceManagerRuntimeServicesConfiguration configuration,
		HighAvailabilityServices highAvailabilityServices,
		ScheduledExecutor scheduledExecutor) throws Exception {

	final SlotManagerConfiguration slotManagerConfiguration = configuration.getSlotManagerConfiguration();

	final SlotManager slotManager = new SlotManagerImpl(
		scheduledExecutor,
		slotManagerConfiguration.getTaskManagerRequestTimeout(),
		slotManagerConfiguration.getSlotRequestTimeout(),
		slotManagerConfiguration.getTaskManagerTimeout(),
		slotManagerConfiguration.isWaitResultConsumedBeforeRelease());

	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		configuration.getJobTimeout());

	return new ResourceManagerRuntimeServices(slotManager, jobLeaderIdService);
}
 
Example #15
Source File: DefaultJobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void start(
	final String initialOwnerAddress,
	final RpcService initialRpcService,
	final HighAvailabilityServices initialHighAvailabilityServices,
	final JobLeaderListener initialJobLeaderListener) {

	if (DefaultJobLeaderService.State.CREATED != state) {
		throw new IllegalStateException("The service has already been started.");
	} else {
		LOG.info("Start job leader service.");

		this.ownerAddress = Preconditions.checkNotNull(initialOwnerAddress);
		this.rpcService = Preconditions.checkNotNull(initialRpcService);
		this.highAvailabilityServices = Preconditions.checkNotNull(initialHighAvailabilityServices);
		this.jobLeaderListener = Preconditions.checkNotNull(initialJobLeaderListener);
		state = DefaultJobLeaderService.State.STARTED;
	}
}
 
Example #16
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskManagerRpcServiceShouldBindToIpAddressDeterminedByConnectingToResourceManager() throws Exception {
	final ServerSocket testJobManagerSocket = openServerSocket();
	final Configuration config = createFlinkConfigWithJobManagerPort(testJobManagerSocket.getLocalPort());
	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	RpcService taskManagerRpcService = null;
	try {
		taskManagerRpcService = TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		assertThat(taskManagerRpcService.getAddress(), is(ipAddress()));
	} finally {
		maybeCloseRpcService(taskManagerRpcService);
		highAvailabilityServices.closeAndCleanupAllData();
		IOUtils.closeQuietly(testJobManagerSocket);
	}
}
 
Example #17
Source File: TaskManagerRunnerConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskManagerRpcServiceShouldBindToConfiguredTaskManagerHostname() throws Exception {
	final String taskmanagerHost = "testhostname";
	final Configuration config = createFlinkConfigWithPredefinedTaskManagerHostname(taskmanagerHost);
	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	RpcService taskManagerRpcService = null;
	try {
		taskManagerRpcService = TaskManagerRunner.createRpcService(config, highAvailabilityServices);

		assertThat(taskManagerRpcService.getPort(), is(greaterThanOrEqualTo(0)));
		assertThat(taskManagerRpcService.getAddress(), is(equalTo(taskmanagerHost)));
	} finally {
		maybeCloseRpcService(taskManagerRpcService);
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example #18
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
 
Example #19
Source File: ClusterClient.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval
 * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down
 */
public ClusterClient(
		Configuration flinkConfig,
		HighAvailabilityServices highAvailabilityServices,
		boolean sharedHaServices) {
	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);

	this.timeout = AkkaUtils.getClientTimeout(flinkConfig);
	this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);

	this.actorSystemLoader = new LazyActorSystemLoader(
		highAvailabilityServices,
		Time.milliseconds(lookupTimeout.toMillis()),
		flinkConfig,
		log);

	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
	this.sharedHaServices = sharedHaServices;
}
 
Example #20
Source File: StandaloneLeaderElectionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 
Example #21
Source File: TestingDispatcher.java    From flink with Apache License 2.0 5 votes vote down vote up
TestingDispatcher(
	RpcService rpcService,
	String endpointId,
	Configuration configuration,
	HighAvailabilityServices highAvailabilityServices,
	GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
	BlobServer blobServer,
	HeartbeatServices heartbeatServices,
	JobManagerMetricGroup jobManagerMetricGroup,
	@Nullable String metricQueryServiceAddress,
	ArchivedExecutionGraphStore archivedExecutionGraphStore,
	JobManagerRunnerFactory jobManagerRunnerFactory,
	FatalErrorHandler fatalErrorHandler) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		highAvailabilityServices.getSubmittedJobGraphStore(),
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServiceAddress,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler,
		VoidHistoryServerArchivist.INSTANCE);

	this.startFuture = new CompletableFuture<>();
}
 
Example #22
Source File: ResourceManagerFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
JobManagerMetricGroup jobManagerMetricGroup) throws Exception;
 
Example #23
Source File: ZooKeeperLeaderRetrievalTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the LeaderRetrievalUtils.findConnectingAddress stops trying to find the
 * connecting address if no leader address has been specified. The call should return
 * then InetAddress.getLocalHost().
 */
@Test
public void testTimeoutOfFindConnectingAddress() throws Exception {
	FiniteDuration timeout = new FiniteDuration(1L, TimeUnit.SECONDS);

	LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
	InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);

	assertEquals(InetAddress.getLocalHost(), result);
}
 
Example #24
Source File: ActiveResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
protected abstract ResourceManager<T> createActiveResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
JobManagerMetricGroup jobManagerMetricGroup) throws Exception;
 
Example #25
Source File: YarnResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ResourceManager<YarnWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	return new YarnResourceManager(
		rpcService,
		resourceId,
		configuration,
		System.getenv(),
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		resourceManagerMetricGroup);
}
 
Example #26
Source File: MesosResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ResourceManager<RegisteredMesosWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) throws Exception {

	final MesosTaskManagerParameters taskManagerParameters = MesosUtils.createTmParameters(configuration, LOG);
	final ContainerSpecification taskManagerContainerSpec = MesosUtils.createContainerSpec(configuration);

	return new MesosResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		configuration,
		mesosServices,
		schedulerConfiguration,
		taskManagerParameters,
		taskManagerContainerSpec,
		webInterfaceUrl,
		resourceManagerMetricGroup);
}
 
Example #27
Source File: TaskManagerRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create a RPC service for the task manager.
 *
 * @param configuration The configuration for the TaskManager.
 * @param haServices to use for the task manager hostname retrieval
 */
public static RpcService createRpcService(
		final Configuration configuration,
		final HighAvailabilityServices haServices) throws Exception {

	checkNotNull(configuration);
	checkNotNull(haServices);

	final String taskManagerAddress = determineTaskManagerBindAddress(configuration, haServices);
	final String portRangeDefinition = configuration.getString(TaskManagerOptions.RPC_PORT);

	return AkkaRpcServiceUtils.createRpcService(taskManagerAddress, portRangeDefinition, configuration);
}
 
Example #28
Source File: TestingMiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected Collection<? extends DispatcherResourceManagerComponent> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();

	final List<DispatcherResourceManagerComponent> result = new ArrayList<>(numberDispatcherResourceManagerComponents);

	for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) {
		result.add(
			dispatcherResourceManagerComponentFactory.create(
				configuration,
				getIOExecutor(),
				rpcServiceFactory.createRpcService(),
				haServices,
				blobServer,
				heartbeatServices,
				metricRegistry,
				new MemoryArchivedExecutionGraphStore(),
				metricQueryServiceRetriever,
				fatalErrorHandler));
	}

	return result;
}
 
Example #29
Source File: StandaloneResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}
 
Example #30
Source File: ClusterClient.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private LazyActorSystemLoader(
		HighAvailabilityServices highAvailabilityServices,
		Time timeout,
		Configuration configuration,
		Logger log) {
	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.configuration = Preconditions.checkNotNull(configuration);
	this.log = Preconditions.checkNotNull(log);
}