Java Code Examples for org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils

The following examples show how to use org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: StandaloneUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example 2
Source Project: flink   Source File: StandaloneUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example 3
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a RPC service for the task manager.
 *
 * @param configuration The configuration for the TaskManager.
 * @param haServices to use for the task manager hostname retrieval
 */
@VisibleForTesting
static RpcService createRpcService(
	final Configuration configuration,
	final HighAvailabilityServices haServices) throws Exception {

	checkNotNull(configuration);
	checkNotNull(haServices);

	return AkkaRpcServiceUtils.createRemoteRpcService(
		configuration,
		determineTaskManagerBindAddress(configuration, haServices),
		configuration.getString(TaskManagerOptions.RPC_PORT),
		configuration.getString(TaskManagerOptions.BIND_HOST),
		configuration.getOptional(TaskManagerOptions.RPC_BIND_PORT));
}
 
Example 4
Source Project: Flink-CEPplus   Source File: TaskManagerRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a RPC service for the task manager.
 *
 * @param configuration The configuration for the TaskManager.
 * @param haServices to use for the task manager hostname retrieval
 */
public static RpcService createRpcService(
		final Configuration configuration,
		final HighAvailabilityServices haServices) throws Exception {

	checkNotNull(configuration);
	checkNotNull(haServices);

	final String taskManagerAddress = determineTaskManagerBindAddress(configuration, haServices);
	final String portRangeDefinition = configuration.getString(TaskManagerOptions.RPC_PORT);

	return AkkaRpcServiceUtils.createRpcService(taskManagerAddress, portRangeDefinition, configuration);
}
 
Example 5
Source Project: flink   Source File: MetricUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static RpcService startMetricsRpcService(Configuration configuration, String hostname) throws Exception {
	final String portRange = configuration.getString(MetricOptions.QUERY_SERVICE_PORT);
	final int threadPriority = configuration.getInteger(MetricOptions.QUERY_SERVICE_THREAD_PRIORITY);

	return AkkaRpcServiceUtils.createRpcService(
		hostname,
		portRange,
		configuration,
		METRICS_ACTOR_SYSTEM_NAME,
		new BootstrapTools.FixedThreadPoolExecutorConfiguration(1, 1, threadPriority));
}
 
Example 6
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a RPC service for the task manager.
 *
 * @param configuration The configuration for the TaskManager.
 * @param haServices to use for the task manager hostname retrieval
 */
public static RpcService createRpcService(
		final Configuration configuration,
		final HighAvailabilityServices haServices) throws Exception {

	checkNotNull(configuration);
	checkNotNull(haServices);

	final String taskManagerAddress = determineTaskManagerBindAddress(configuration, haServices);
	final String portRangeDefinition = configuration.getString(TaskManagerOptions.RPC_PORT);

	return AkkaRpcServiceUtils.createRpcService(taskManagerAddress, portRangeDefinition, configuration);
}
 
Example 7
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Factory method to instantiate the remote RPC service.
 *
 * @param configuration Flink configuration.
 * @param bindAddress The address to bind the RPC service to.
 * @param bindPort The port range to bind the RPC service to.
 * @return The instantiated RPC service
 */
protected RpcService createRemoteRpcService(
		Configuration configuration,
		String bindAddress,
		int bindPort) throws Exception {
	return AkkaRpcServiceUtils.remoteServiceBuilder(configuration, bindAddress, String.valueOf(bindPort))
		.withBindAddress(bindAddress)
		.withBindPort(bindPort)
		.withCustomConfig(AkkaUtils.testDispatcherConfig())
		.createAndStart();
}
 
Example 8
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Factory method to instantiate the remote RPC service.
 *
 * @param configuration Flink configuration.
 * @param externalAddress The external address to access the RPC service.
 * @param externalPortRange The external port range to access the RPC service.
 * @param bindAddress The address to bind the RPC service to.
 * @return The instantiated RPC service
 */
protected RpcService createRemoteRpcService(
	Configuration configuration,
	String externalAddress,
	String externalPortRange,
	String bindAddress) throws Exception {
	return AkkaRpcServiceUtils.remoteServiceBuilder(configuration, externalAddress, externalPortRange)
		.withBindAddress(bindAddress)
		.withCustomConfig(AkkaUtils.testDispatcherConfig())
		.createAndStart();
}
 
Example 9
Source Project: flink   Source File: Dispatcher.java    License: Apache License 2.0 5 votes vote down vote up
public Dispatcher(
		RpcService rpcService,
		DispatcherId fencingToken,
		DispatcherBootstrap dispatcherBootstrap,
		DispatcherServices dispatcherServices) throws Exception {
	super(rpcService, AkkaRpcServiceUtils.createRandomName(DISPATCHER_NAME), fencingToken);
	checkNotNull(dispatcherServices);

	this.configuration = dispatcherServices.getConfiguration();
	this.highAvailabilityServices = dispatcherServices.getHighAvailabilityServices();
	this.resourceManagerGatewayRetriever = dispatcherServices.getResourceManagerGatewayRetriever();
	this.heartbeatServices = dispatcherServices.getHeartbeatServices();
	this.blobServer = dispatcherServices.getBlobServer();
	this.fatalErrorHandler = dispatcherServices.getFatalErrorHandler();
	this.jobGraphWriter = dispatcherServices.getJobGraphWriter();
	this.jobManagerMetricGroup = dispatcherServices.getJobManagerMetricGroup();
	this.metricServiceQueryAddress = dispatcherServices.getMetricQueryServiceAddress();

	this.jobManagerSharedServices = JobManagerSharedServices.fromConfiguration(
		configuration,
		blobServer,
		fatalErrorHandler);

	this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry();

	jobManagerRunnerFutures = new HashMap<>(16);

	this.historyServerArchivist = dispatcherServices.getHistoryServerArchivist();

	this.archivedExecutionGraphStore = dispatcherServices.getArchivedExecutionGraphStore();

	this.jobManagerRunnerFactory = dispatcherServices.getJobManagerRunnerFactory();

	this.jobManagerTerminationFutures = new HashMap<>(2);

	this.shutDownFuture = new CompletableFuture<>();

	this.dispatcherBootstrap = checkNotNull(dispatcherBootstrap);
}
 
Example 10
Source Project: flink   Source File: MetricUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static RpcService startMetricRpcService(
		Configuration configuration, AkkaRpcServiceUtils.AkkaRpcServiceBuilder rpcServiceBuilder) throws Exception {
	final int threadPriority = configuration.getInteger(MetricOptions.QUERY_SERVICE_THREAD_PRIORITY);

	return rpcServiceBuilder
		.withActorSystemName(METRICS_ACTOR_SYSTEM_NAME)
		.withActorSystemExecutorConfiguration(new BootstrapTools.FixedThreadPoolExecutorConfiguration(1, 1, threadPriority))
		.createAndStart();
}
 
Example 11
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 5 votes vote down vote up
protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService(
				configuration,
				configuration.getString(JobManagerOptions.ADDRESS),
				getRPCPortRange(configuration),
				configuration.getString(JobManagerOptions.BIND_HOST),
				configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				ClusterEntrypointUtils.getPoolSize(configuration),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration, pluginManager);

			final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress());
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			final String hostname = RpcUtils.getHostname(commonRpcService);

			processMetricGroup = MetricUtils.instantiateProcessMetricGroup(
				metricRegistry,
				hostname,
				ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
 
Example 12
Source Project: flink   Source File: AddressResolutionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNoAddressResolution() throws UnknownHostException {
	AkkaRpcServiceUtils.getRpcUrl(
		NON_EXISTING_HOSTNAME,
		PORT,
		ENDPOINT_NAME,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
		new Configuration());
}
 
Example 13
Source Project: flink   Source File: AddressResolutionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTryAddressResolution() {
	try {
		AkkaRpcServiceUtils.getRpcUrl(
			NON_EXISTING_HOSTNAME,
			PORT,
			ENDPOINT_NAME,
			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION,
			new Configuration());
		fail("This should fail with an UnknownHostException");
	} catch (UnknownHostException ignore) {
		// expected
	}
}
 
Example 14
/**
 * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration.
 * This constructor parses the ResourceManager address from the Flink configuration and sets
 * up the HDFS access to store recovery data in the YARN application's working directory.
 *
 * @param config     The Flink configuration of this component / process.
 * @param hadoopConf The Hadoop configuration for the YARN cluster.
 *
 * @throws IOException
 *             Thrown, if the initialization of the Hadoop file system used by YARN fails.
 * @throws IllegalConfigurationException
 *             Thrown, if the Flink configuration does not properly describe the ResourceManager address and port.
 */
public YarnPreConfiguredMasterNonHaServices(
		Configuration config,
		org.apache.hadoop.conf.Configuration hadoopConf,
		HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {

	super(config, hadoopConf);

	// track whether we successfully perform the initialization
	boolean successful = false;

	try {
		// extract the hostname and port of the resource manager
		final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS);
		final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT);

		if (rmHost == null) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing.");
		}
		if (rmPort < 0) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing.");
		}
		if (rmPort <= 0 || rmPort >= 65536) {
			throw new IllegalConfigurationException("Invalid value for '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
		}

		this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			ResourceManager.RESOURCE_MANAGER_NAME,
			addressResolution,
			config);

		this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			Dispatcher.DISPATCHER_NAME,
			addressResolution,
			config);

		// all well!
		successful = true;
	}
	finally {
		if (!successful) {
			// quietly undo what the parent constructor initialized
			try {
				super.close();
			} catch (Throwable ignored) {}
		}
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: JobMaster.java    License: Apache License 2.0 4 votes vote down vote up
public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME));

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
			jobGraph.getSerializedExecutionConfig()
					.deserializeValue(userCodeLoader)
					.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		jobManagerSharedServices.getRestartStrategyFactory(),
		jobGraph.isCheckpointingEnabled());

	log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());
	this.lastInternalSavepoint = null;

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
}
 
Example 16
Source Project: Flink-CEPplus   Source File: TaskExecutor.java    License: Apache License 2.0 4 votes vote down vote up
public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServicePath = metricQueryServicePath;

	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobManagerTable = taskExecutorServices.getJobManagerTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.taskManagerLocation = taskExecutorServices.getTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.networkEnvironment = taskExecutorServices.getNetworkEnvironment();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.jobManagerConnections = new HashMap<>(4);

	this.hardwareDescription = HardwareDescription.extractFromSystem(
		taskExecutorServices.getMemoryManager().getMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	this.stackTraceSampleService = new StackTraceSampleService(rpcService.getScheduledExecutor());
}
 
Example 17
Source Project: Flink-CEPplus   Source File: ClusterEntrypoint.java    License: Apache License 2.0 4 votes vote down vote up
@Nonnull
private RpcService createRpcService(Configuration configuration, String bindAddress, String portRange) throws Exception {
	return AkkaRpcServiceUtils.createRpcService(bindAddress, portRange, configuration);
}
 
Example 18
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 19
/**
 * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration.
 * This constructor parses the ResourceManager address from the Flink configuration and sets
 * up the HDFS access to store recovery data in the YARN application's working directory.
 *
 * @param config     The Flink configuration of this component / process.
 * @param hadoopConf The Hadoop configuration for the YARN cluster.
 *
 * @throws IOException
 *             Thrown, if the initialization of the Hadoop file system used by YARN fails.
 * @throws IllegalConfigurationException
 *             Thrown, if the Flink configuration does not properly describe the ResourceManager address and port.
 */
public YarnPreConfiguredMasterNonHaServices(
		Configuration config,
		org.apache.hadoop.conf.Configuration hadoopConf,
		HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {

	super(config, hadoopConf);

	// track whether we successfully perform the initialization
	boolean successful = false;

	try {
		// extract the hostname and port of the resource manager
		final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS);
		final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT);

		if (rmHost == null) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing.");
		}
		if (rmPort < 0) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing.");
		}
		if (rmPort <= 0 || rmPort >= 65536) {
			throw new IllegalConfigurationException("Invalid value for '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
		}

		this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			ResourceManager.RESOURCE_MANAGER_NAME,
			addressResolution,
			config);

		this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			Dispatcher.DISPATCHER_NAME,
			addressResolution,
			config);

		// all well!
		successful = true;
	}
	finally {
		if (!successful) {
			// quietly undo what the parent constructor initialized
			try {
				super.close();
			} catch (Throwable ignored) {}
		}
	}
}
 
Example 20
Source Project: flink   Source File: JobMaster.java    License: Apache License 2.0 4 votes vote down vote up
public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader,
		SchedulerNGFactory schedulerNGFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTrackerFactory partitionTrackerFactory) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME));

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.schedulerNGFactory = checkNotNull(schedulerNGFactory);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);
	this.partitionTracker = checkNotNull(partitionTrackerFactory)
		.create(resourceID -> {
			Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManagerInfo = registeredTaskManagers.get(resourceID);
			if (taskManagerInfo == null) {
				return Optional.empty();
			}

			return Optional.of(taskManagerInfo.f1);
		});

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.schedulerNG = createScheduler(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
}
 
Example 21
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 4 votes vote down vote up
public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler,
		PartitionTable<JobID> partitionTable) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.partitionTable = partitionTable;
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServiceAddress = metricQueryServiceAddress;

	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobManagerTable = taskExecutorServices.getJobManagerTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.taskManagerLocation = taskExecutorServices.getTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.shuffleEnvironment = taskExecutorServices.getShuffleEnvironment();
	this.kvStateService = taskExecutorServices.getKvStateService();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.jobManagerConnections = new HashMap<>(4);

	this.hardwareDescription = HardwareDescription.extractFromSystem(
		taskExecutorServices.getMemoryManager().getMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	this.stackTraceSampleService = new StackTraceSampleService(rpcService.getScheduledExecutor());
}
 
Example 22
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 4 votes vote down vote up
@Nonnull
private RpcService createRpcService(Configuration configuration, String bindAddress, String portRange) throws Exception {
	return AkkaRpcServiceUtils.createRpcService(bindAddress, portRange, configuration);
}
 
Example 23
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 24
Source Project: flink   Source File: MetricUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static RpcService startRemoteMetricsRpcService(Configuration configuration, String hostname) throws Exception {
	final String portRange = configuration.getString(MetricOptions.QUERY_SERVICE_PORT);

	return startMetricRpcService(configuration, AkkaRpcServiceUtils.remoteServiceBuilder(configuration, hostname, portRange));
}
 
Example 25
Source Project: flink   Source File: MetricUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static RpcService startLocalMetricsRpcService(Configuration configuration) throws Exception {
	return startMetricRpcService(configuration, AkkaRpcServiceUtils.localServiceBuilder(configuration));
}
 
Example 26
Source Project: flink   Source File: JobMaster.java    License: Apache License 2.0 4 votes vote down vote up
public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader,
		SchedulerNGFactory schedulerNGFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTrackerFactory partitionTrackerFactory) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME), null);

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.schedulerNGFactory = checkNotNull(schedulerNGFactory);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);
	this.partitionTracker = checkNotNull(partitionTrackerFactory)
		.create(resourceID -> {
			Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManagerInfo = registeredTaskManagers.get(resourceID);
			if (taskManagerInfo == null) {
				return Optional.empty();
			}

			return Optional.of(taskManagerInfo.f1);
		});

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.schedulerNG = createScheduler(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
	this.taskManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
	this.resourceManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
}
 
Example 27
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 4 votes vote down vote up
public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		ExternalResourceInfoProvider externalResourceInfoProvider,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler,
		TaskExecutorPartitionTracker partitionTracker,
		BackPressureSampleService backPressureSampleService) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.partitionTracker = partitionTracker;
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServiceAddress = metricQueryServiceAddress;
	this.backPressureSampleService = checkNotNull(backPressureSampleService);
	this.externalResourceInfoProvider = checkNotNull(externalResourceInfoProvider);

	this.libraryCacheManager = taskExecutorServices.getLibraryCacheManager();
	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobTable = taskExecutorServices.getJobTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.unresolvedTaskManagerLocation = taskExecutorServices.getUnresolvedTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.shuffleEnvironment = taskExecutorServices.getShuffleEnvironment();
	this.kvStateService = taskExecutorServices.getKvStateService();
	this.ioExecutor = taskExecutorServices.getIOExecutor();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.hardwareDescription = HardwareDescription.extractFromSystem(taskExecutorServices.getManagedMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	final ResourceID resourceId = taskExecutorServices.getUnresolvedTaskManagerLocation().getResourceID();
	this.jobManagerHeartbeatManager = createJobManagerHeartbeatManager(heartbeatServices, resourceId);
	this.resourceManagerHeartbeatManager = createResourceManagerHeartbeatManager(heartbeatServices, resourceId);
}
 
Example 28
Source Project: flink   Source File: ResourceManager.java    License: Apache License 2.0 4 votes vote down vote up
public ResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		Time rpcTimeout) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(RESOURCE_MANAGER_NAME), null);

	this.resourceId = checkNotNull(resourceId);
	this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.slotManager = checkNotNull(slotManager);
	this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
	this.clusterInformation = checkNotNull(clusterInformation);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.resourceManagerMetricGroup = checkNotNull(resourceManagerMetricGroup);

	this.jobManagerRegistrations = new HashMap<>(4);
	this.jmResourceIdRegistrations = new HashMap<>(4);
	this.taskExecutors = new HashMap<>(8);
	this.taskExecutorGatewayFutures = new HashMap<>(8);

	this.jobManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
	this.taskManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();

	this.clusterPartitionTracker = checkNotNull(clusterPartitionTrackerFactory).get(
		(taskExecutorResourceId, dataSetIds) -> taskExecutors.get(taskExecutorResourceId).getTaskExecutorGateway()
			.releaseClusterPartitions(dataSetIds, rpcTimeout)
			.exceptionally(throwable -> {
				log.debug("Request for release of cluster partitions belonging to data sets {} was not successful.", dataSetIds, throwable);
				throw new CompletionException(throwable);
			})
	);
}
 
Example 29
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(ResourceManager.RESOURCE_MANAGER_NAME),
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(Dispatcher.DISPATCHER_NAME),
				addressResolution,
				configuration);
			final String webMonitorAddress = getWebMonitorAddress(
				configuration,
				addressResolution);

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				webMonitorAddress);
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 30
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Factory method to instantiate the local RPC service.
 *
 * @param configuration Flink configuration.
 * @return The instantiated RPC service
 */
protected RpcService createLocalRpcService(Configuration configuration) throws Exception {
	return AkkaRpcServiceUtils.localServiceBuilder(configuration)
		.withCustomConfig(AkkaUtils.testDispatcherConfig())
		.createAndStart();
}