org.apache.flink.runtime.resourcemanager.ResourceManager Java Examples

The following examples show how to use org.apache.flink.runtime.resourcemanager.ResourceManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SessionDispatcherResourceManagerComponentFactory.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected DispatcherResourceManagerComponent<Dispatcher> createDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new SessionDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}

Example #2

Source File: JobDispatcherResourceManagerComponent.java From Flink-CEPplus with Apache License 2.0

6 votes

JobDispatcherResourceManagerComponent(
		MiniDispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup);

	final CompletableFuture<ApplicationStatus> shutDownFuture = getShutDownFuture();

	dispatcher.getJobTerminationFuture().whenComplete((applicationStatus, throwable) -> {
		if (throwable != null) {
			shutDownFuture.completeExceptionally(throwable);
		} else {
			shutDownFuture.complete(applicationStatus);
		}
	});
}

Example #3

Source File: DispatcherResourceManagerComponent.java From Flink-CEPplus with Apache License 2.0

6 votes

DispatcherResourceManagerComponent(
		@Nonnull T dispatcher,
		@Nonnull ResourceManager<?> resourceManager,
		@Nonnull LeaderRetrievalService dispatcherLeaderRetrievalService,
		@Nonnull LeaderRetrievalService resourceManagerRetrievalService,
		@Nonnull WebMonitorEndpoint<?> webMonitorEndpoint,
		@Nonnull JobManagerMetricGroup jobManagerMetricGroup) {
	this.resourceManager = resourceManager;
	this.dispatcher = dispatcher;
	this.dispatcherLeaderRetrievalService = dispatcherLeaderRetrievalService;
	this.resourceManagerRetrievalService = resourceManagerRetrievalService;
	this.webMonitorEndpoint = webMonitorEndpoint;
	this.jobManagerMetricGroup = jobManagerMetricGroup;
	this.terminationFuture = new CompletableFuture<>();
	this.shutDownFuture = new CompletableFuture<>();

	registerShutDownFuture();
}

Example #4

Source File: JobDispatcherResourceManagerComponentFactory.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected DispatcherResourceManagerComponent<MiniDispatcher> createDispatcherResourceManagerComponent(
		MiniDispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new JobDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}

Example #5

Source File: DispatcherResourceManagerComponent.java From flink with Apache License 2.0

6 votes

DispatcherResourceManagerComponent(
		@Nonnull DispatcherRunner dispatcherRunner,
		@Nonnull ResourceManager<?> resourceManager,
		@Nonnull LeaderRetrievalService dispatcherLeaderRetrievalService,
		@Nonnull LeaderRetrievalService resourceManagerRetrievalService,
		@Nonnull WebMonitorEndpoint<?> webMonitorEndpoint) {
	this.dispatcherRunner = dispatcherRunner;
	this.resourceManager = resourceManager;
	this.dispatcherLeaderRetrievalService = dispatcherLeaderRetrievalService;
	this.resourceManagerRetrievalService = resourceManagerRetrievalService;
	this.webMonitorEndpoint = webMonitorEndpoint;
	this.terminationFuture = new CompletableFuture<>();
	this.shutDownFuture = new CompletableFuture<>();

	registerShutDownFuture();
}

Example #6

Source File: JobDispatcherResourceManagerComponentFactory.java From flink with Apache License 2.0

6 votes

@Override
protected DispatcherResourceManagerComponent<MiniDispatcher> createDispatcherResourceManagerComponent(
		MiniDispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new JobDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}

Example #7

Source File: DispatcherResourceManagerComponent.java From flink with Apache License 2.0

6 votes

DispatcherResourceManagerComponent(
		@Nonnull T dispatcher,
		@Nonnull ResourceManager<?> resourceManager,
		@Nonnull LeaderRetrievalService dispatcherLeaderRetrievalService,
		@Nonnull LeaderRetrievalService resourceManagerRetrievalService,
		@Nonnull WebMonitorEndpoint<?> webMonitorEndpoint,
		@Nonnull JobManagerMetricGroup jobManagerMetricGroup) {
	this.resourceManager = resourceManager;
	this.dispatcher = dispatcher;
	this.dispatcherLeaderRetrievalService = dispatcherLeaderRetrievalService;
	this.resourceManagerRetrievalService = resourceManagerRetrievalService;
	this.webMonitorEndpoint = webMonitorEndpoint;
	this.jobManagerMetricGroup = jobManagerMetricGroup;
	this.terminationFuture = new CompletableFuture<>();
	this.shutDownFuture = new CompletableFuture<>();

	registerShutDownFuture();
}

Example #8

Source File: JobDispatcherResourceManagerComponent.java From flink with Apache License 2.0

6 votes

JobDispatcherResourceManagerComponent(
		MiniDispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup);

	final CompletableFuture<ApplicationStatus> shutDownFuture = getShutDownFuture();

	dispatcher.getJobTerminationFuture().whenComplete((applicationStatus, throwable) -> {
		if (throwable != null) {
			shutDownFuture.completeExceptionally(throwable);
		} else {
			shutDownFuture.complete(applicationStatus);
		}
	});
}

Example #9

Source File: SessionDispatcherResourceManagerComponentFactory.java From flink with Apache License 2.0

6 votes

@Override
protected DispatcherResourceManagerComponent<Dispatcher> createDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new SessionDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}

Example #10

Source File: YarnResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<YarnWorkerNode> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	final ResourceManagerRuntimeServicesConfiguration rmServicesConfiguration = ResourceManagerRuntimeServicesConfiguration.fromConfiguration(configuration);
	final ResourceManagerRuntimeServices rmRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration(
		rmServicesConfiguration,
		highAvailabilityServices,
		rpcService.getScheduledExecutor());

	return new YarnResourceManager(
		rpcService,
		getEndpointId(),
		resourceId,
		configuration,
		System.getenv(),
		highAvailabilityServices,
		heartbeatServices,
		rmRuntimeServices.getSlotManager(),
		metricRegistry,
		rmRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
}

Example #11

Source File: KubernetesResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<KubernetesWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final KubernetesResourceManagerConfiguration kubernetesResourceManagerConfiguration =
		new KubernetesResourceManagerConfiguration(
			configuration.getString(KubernetesConfigOptions.CLUSTER_ID),
			POD_CREATION_RETRY_INTERVAL);

	return new KubernetesResourceManager(
		rpcService,
		resourceId,
		configuration,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		KubeClientFactory.fromConfiguration(configuration),
		kubernetesResourceManagerConfiguration);
}

Example #12

Source File: MesosResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<RegisteredMesosWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) throws Exception {

	final MesosTaskManagerParameters taskManagerParameters = MesosUtils.createTmParameters(configuration, LOG);
	final ContainerSpecification taskManagerContainerSpec = MesosUtils.createContainerSpec(configuration);

	return new MesosResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		configuration,
		mesosServices,
		schedulerConfiguration,
		taskManagerParameters,
		taskManagerContainerSpec,
		webInterfaceUrl,
		resourceManagerMetricGroup);
}

Example #13

Source File: AbstractDispatcherResourceManagerComponentFactory.java From flink with Apache License 2.0

5 votes

protected abstract DispatcherResourceManagerComponent<T> createDispatcherResourceManagerComponent(
T dispatcher,
ResourceManager<?> resourceManager,
LeaderRetrievalService dispatcherLeaderRetrievalService,
LeaderRetrievalService resourceManagerRetrievalService,
WebMonitorEndpoint<?> webMonitorEndpoint,
JobManagerMetricGroup jobManagerMetricGroup);

Example #14

Source File: SessionDispatcherResourceManagerComponent.java From flink with Apache License 2.0

5 votes

SessionDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup);
}

Example #15

Source File: YarnResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<YarnWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	return new YarnResourceManager(
		rpcService,
		resourceId,
		configuration,
		System.getenv(),
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		resourceManagerMetricGroup);
}

Example #16

Source File: MesosResourceManagerFactory.java From flink with Apache License 2.0

5 votes

@Override
public ResourceManager<RegisteredMesosWorkerNode> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	final ResourceManagerRuntimeServicesConfiguration rmServicesConfiguration = ResourceManagerRuntimeServicesConfiguration.fromConfiguration(configuration);
	final ResourceManagerRuntimeServices rmRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration(
		rmServicesConfiguration,
		highAvailabilityServices,
		rpcService.getScheduledExecutor());

	return new MesosResourceManager(
		rpcService,
		getEndpointId(),
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		rmRuntimeServices.getSlotManager(),
		metricRegistry,
		rmRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		configuration,
		mesosServices,
		schedulerConfiguration,
		taskManagerParameters,
		taskManagerContainerSpec,
		webInterfaceUrl,
		jobManagerMetricGroup);
}

Example #17

Source File: AbstractDispatcherResourceManagerComponentFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

protected abstract DispatcherResourceManagerComponent<T> createDispatcherResourceManagerComponent(
T dispatcher,
ResourceManager<?> resourceManager,
LeaderRetrievalService dispatcherLeaderRetrievalService,
LeaderRetrievalService resourceManagerRetrievalService,
WebMonitorEndpoint<?> webMonitorEndpoint,
JobManagerMetricGroup jobManagerMetricGroup);

Example #18

Source File: SessionDispatcherResourceManagerComponent.java From Flink-CEPplus with Apache License 2.0

5 votes

SessionDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup);
}

Example #19

Source File: MesosResourceManagerFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public ResourceManager<RegisteredMesosWorkerNode> createResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) throws Exception {
	final ResourceManagerRuntimeServicesConfiguration rmServicesConfiguration = ResourceManagerRuntimeServicesConfiguration.fromConfiguration(configuration);
	final ResourceManagerRuntimeServices rmRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration(
		rmServicesConfiguration,
		highAvailabilityServices,
		rpcService.getScheduledExecutor());

	return new MesosResourceManager(
		rpcService,
		getEndpointId(),
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		rmRuntimeServices.getSlotManager(),
		metricRegistry,
		rmRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		configuration,
		mesosServices,
		schedulerConfiguration,
		taskManagerParameters,
		taskManagerContainerSpec,
		webInterfaceUrl,
		jobManagerMetricGroup);
}

Example #20

Source File: HighAvailabilityServicesUtils.java From flink with Apache License 2.0

4 votes

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(ResourceManager.RESOURCE_MANAGER_NAME),
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(Dispatcher.DISPATCHER_NAME),
				addressResolution,
				configuration);
			final String webMonitorAddress = getWebMonitorAddress(
				configuration,
				addressResolution);

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				webMonitorAddress);
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}

Example #21

Source File: HighAvailabilityServicesUtils.java From flink with Apache License 2.0

4 votes

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}

Example #22

Source File: YarnPreConfiguredMasterNonHaServices.java From flink with Apache License 2.0

4 votes

/**
 * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration.
 * This constructor parses the ResourceManager address from the Flink configuration and sets
 * up the HDFS access to store recovery data in the YARN application's working directory.
 *
 * @param config     The Flink configuration of this component / process.
 * @param hadoopConf The Hadoop configuration for the YARN cluster.
 *
 * @throws IOException
 *             Thrown, if the initialization of the Hadoop file system used by YARN fails.
 * @throws IllegalConfigurationException
 *             Thrown, if the Flink configuration does not properly describe the ResourceManager address and port.
 */
public YarnPreConfiguredMasterNonHaServices(
		Configuration config,
		org.apache.hadoop.conf.Configuration hadoopConf,
		HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {

	super(config, hadoopConf);

	// track whether we successfully perform the initialization
	boolean successful = false;

	try {
		// extract the hostname and port of the resource manager
		final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS);
		final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT);

		if (rmHost == null) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing.");
		}
		if (rmPort < 0) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing.");
		}
		if (rmPort <= 0 || rmPort >= 65536) {
			throw new IllegalConfigurationException("Invalid value for '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
		}

		this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			ResourceManager.RESOURCE_MANAGER_NAME,
			addressResolution,
			config);

		this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			Dispatcher.DISPATCHER_NAME,
			addressResolution,
			config);

		// all well!
		successful = true;
	}
	finally {
		if (!successful) {
			// quietly undo what the parent constructor initialized
			try {
				super.close();
			} catch (Throwable ignored) {}
		}
	}
}

Example #23

Source File: HighAvailabilityServicesUtils.java From Flink-CEPplus with Apache License 2.0

4 votes

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}

Example #24

Source File: YarnPreConfiguredMasterNonHaServices.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration.
 * This constructor parses the ResourceManager address from the Flink configuration and sets
 * up the HDFS access to store recovery data in the YARN application's working directory.
 *
 * @param config     The Flink configuration of this component / process.
 * @param hadoopConf The Hadoop configuration for the YARN cluster.
 *
 * @throws IOException
 *             Thrown, if the initialization of the Hadoop file system used by YARN fails.
 * @throws IllegalConfigurationException
 *             Thrown, if the Flink configuration does not properly describe the ResourceManager address and port.
 */
public YarnPreConfiguredMasterNonHaServices(
		Configuration config,
		org.apache.hadoop.conf.Configuration hadoopConf,
		HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {

	super(config, hadoopConf);

	// track whether we successfully perform the initialization
	boolean successful = false;

	try {
		// extract the hostname and port of the resource manager
		final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS);
		final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT);

		if (rmHost == null) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing.");
		}
		if (rmPort < 0) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing.");
		}
		if (rmPort <= 0 || rmPort >= 65536) {
			throw new IllegalConfigurationException("Invalid value for '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
		}

		this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			ResourceManager.RESOURCE_MANAGER_NAME,
			addressResolution,
			config);

		this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			Dispatcher.DISPATCHER_NAME,
			addressResolution,
			config);

		// all well!
		successful = true;
	}
	finally {
		if (!successful) {
			// quietly undo what the parent constructor initialized
			try {
				super.close();
			} catch (Throwable ignored) {}
		}
	}
}