org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils Java Examples

The following examples show how to use org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YarnHighAvailabilityServices.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example #2
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param akkaProtocol True, if security/encryption is enabled, false otherwise.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
		String hostname,
		int port,
		String endpointName,
		HighAvailabilityServicesUtils.AddressResolution addressResolution,
		AkkaProtocol akkaProtocol) throws UnknownHostException {

	checkNotNull(hostname, "hostname is null");
	checkNotNull(endpointName, "endpointName is null");
	checkArgument(isValidClientPort(port), "port must be in [1, 65535]");

	if (addressResolution == AddressResolution.TRY_ADDRESS_RESOLUTION) {
		// Fail fast if the hostname cannot be resolved
		//noinspection ResultOfMethodCallIgnored
		InetAddress.getByName(hostname);
	}

	final String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(hostname, port);

	return internalRpcUrl(endpointName, Optional.of(new RemoteAddressInformation(hostPort, akkaProtocol)));
}
 
Example #3
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #4
Source File: StandaloneUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example #5
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param akkaProtocol True, if security/encryption is enabled, false otherwise.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
		String hostname,
		int port,
		String endpointName,
		HighAvailabilityServicesUtils.AddressResolution addressResolution,
		AkkaProtocol akkaProtocol) throws UnknownHostException {

	checkNotNull(hostname, "hostname is null");
	checkNotNull(endpointName, "endpointName is null");
	checkArgument(port > 0 && port <= 65535, "port must be in [1, 65535]");

	final String protocolPrefix = akkaProtocol == AkkaProtocol.SSL_TCP ? AKKA_SSL_TCP : AKKA_TCP;

	if (addressResolution == AddressResolution.TRY_ADDRESS_RESOLUTION) {
		// Fail fast if the hostname cannot be resolved
		//noinspection ResultOfMethodCallIgnored
		InetAddress.getByName(hostname);
	}

	final String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(hostname, port);

	return String.format("%s://flink@%s/user/%s", protocolPrefix, hostPort, endpointName);
}
 
Example #6
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #7
Source File: YarnHighAvailabilityServices.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example #8
Source File: AkkaRpcServiceUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param akkaProtocol True, if security/encryption is enabled, false otherwise.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
		String hostname,
		int port,
		String endpointName,
		HighAvailabilityServicesUtils.AddressResolution addressResolution,
		AkkaProtocol akkaProtocol) throws UnknownHostException {

	checkNotNull(hostname, "hostname is null");
	checkNotNull(endpointName, "endpointName is null");
	checkArgument(port > 0 && port <= 65535, "port must be in [1, 65535]");

	final String protocolPrefix = akkaProtocol == AkkaProtocol.SSL_TCP ? AKKA_SSL_TCP : AKKA_TCP;

	if (addressResolution == AddressResolution.TRY_ADDRESS_RESOLUTION) {
		// Fail fast if the hostname cannot be resolved
		//noinspection ResultOfMethodCallIgnored
		InetAddress.getByName(hostname);
	}

	final String hostPort = NetUtils.unresolvedHostAndPortToNormalizedString(hostname, port);

	return String.format("%s://flink@%s/user/%s", protocolPrefix, hostPort, endpointName);
}
 
Example #9
Source File: AkkaRpcServiceUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #10
Source File: StandaloneUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example #11
Source File: KubernetesClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private ClusterClientProvider<String> createClusterClientProvider(String clusterId) {
	return () -> {
		final Configuration configuration = new Configuration(flinkConfig);

		final Optional<Endpoint> restEndpoint = client.getRestEndpoint(clusterId);

		if (restEndpoint.isPresent()) {
			configuration.setString(RestOptions.ADDRESS, restEndpoint.get().getAddress());
			configuration.setInteger(RestOptions.PORT, restEndpoint.get().getPort());
		} else {
			throw new RuntimeException(
					new ClusterRetrieveException(
							"Could not get the rest endpoint of " + clusterId));
		}

		try {
			// Flink client will always use Kubernetes service to contact with jobmanager. So we have a pre-configured web
			// monitor address. Using StandaloneClientHAServices to create RestClusterClient is reasonable.
			return new RestClusterClient<>(
				configuration,
				clusterId,
				new StandaloneClientHAServices(HighAvailabilityServicesUtils.getWebMonitorAddress(
					configuration, HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION)));
		} catch (Exception e) {
			client.handleException(e);
			throw new RuntimeException(new ClusterRetrieveException("Could not create the RestClusterClient.", e));
		}
	};
}
 
Example #12
Source File: RestClusterClient.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
RestClusterClient(
	Configuration configuration,
	@Nullable RestClient restClient,
	T clusterId,
	WaitStrategy waitStrategy) throws Exception {
	this(
		configuration,
		restClient,
		clusterId,
		waitStrategy,
		HighAvailabilityServicesUtils.createClientHAService(configuration));
}
 
Example #13
Source File: AddressResolutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTryAddressResolution() {
	try {
		AkkaRpcServiceUtils.getRpcUrl(
			NON_EXISTING_HOSTNAME,
			PORT,
			ENDPOINT_NAME,
			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION,
			new Configuration());
		fail("This should fail with an UnknownHostException");
	} catch (UnknownHostException ignore) {
		// expected
	}
}
 
Example #14
Source File: AddressResolutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoAddressResolution() throws UnknownHostException {
	AkkaRpcServiceUtils.getRpcUrl(
		NON_EXISTING_HOSTNAME,
		PORT,
		ENDPOINT_NAME,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION,
		new Configuration());
}
 
Example #15
Source File: ZooKeeperDefaultDispatcherRunnerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
	fatalErrorHandler = new TestingFatalErrorHandler();
	configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath());

	clusterHaStorageDir = new File(HighAvailabilityServicesUtils.getClusterHighAvailableStoragePath(configuration).toString());
	blobServer = new BlobServer(configuration, BlobUtils.createBlobStoreFromConfig(configuration));
}
 
Example #16
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up
protected HighAvailabilityServices createHaServices(
	Configuration configuration,
	Executor executor) throws Exception {
	return HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
}
 
Example #17
Source File: BlobUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
private static BlobStoreService createFileSystemBlobStore(Configuration configuration) throws IOException {
	final Path clusterStoragePath = HighAvailabilityServicesUtils.getClusterHighAvailableStoragePath(configuration);

	final FileSystem fileSystem;
	try {
		fileSystem = clusterStoragePath.getFileSystem();
	} catch (Exception e) {
		throw new IOException(
			String.format("Could not create FileSystem for highly available storage path (%s)", clusterStoragePath),
			e);
	}

	return new FileSystemBlobStore(fileSystem, clusterStoragePath.toUri().toString());
}
 
Example #18
Source File: MiniCluster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected HighAvailabilityServices createHighAvailabilityServices(Configuration configuration, Executor executor) throws Exception {
	LOG.info("Starting high-availability services");
	return HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
		configuration,
		executor);
}
 
Example #19
Source File: MiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected HighAvailabilityServices createHighAvailabilityServices(Configuration configuration, Executor executor) throws Exception {
	LOG.info("Starting high-availability services");
	return HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
		configuration,
		executor);
}
 
Example #20
Source File: ClusterClient.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 *
 * @throws Exception we cannot create the high availability services
 */
public ClusterClient(Configuration flinkConfig) throws Exception {
	this(
		flinkConfig,
		HighAvailabilityServicesUtils.createHighAvailabilityServices(
			flinkConfig,
			Executors.directExecutor(),
			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION),
		false);
}
 
Example #21
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up
protected HighAvailabilityServices createHaServices(
	Configuration configuration,
	Executor executor) throws Exception {
	return HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
}
 
Example #22
Source File: MiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected HighAvailabilityServices createHighAvailabilityServices(Configuration configuration, Executor executor) throws Exception {
	LOG.info("Starting high-availability services");
	return HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(
		configuration,
		executor);
}
 
Example #23
Source File: ClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
protected HighAvailabilityServices createHaServices(
	Configuration configuration,
	Executor executor) throws Exception {
	return HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
}
 
Example #24
Source File: ClusterClient.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 *
 * @throws Exception we cannot create the high availability services
 */
public ClusterClient(Configuration flinkConfig) throws Exception {
	this(
		flinkConfig,
		HighAvailabilityServicesUtils.createHighAvailabilityServices(
			flinkConfig,
			Executors.directExecutor(),
			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION),
		false);
}
 
Example #25
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private HighAvailabilityServices createHighAvailabilityServices(final Configuration config) throws Exception {
	return HighAvailabilityServicesUtils.createHighAvailabilityServices(
		config,
		Executors.directExecutor(),
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
}
 
Example #26
Source File: TaskManagerRunner.java    From flink with Apache License 2.0 4 votes vote down vote up
public TaskManagerRunner(Configuration configuration, ResourceID resourceId) throws Exception {
	this.configuration = checkNotNull(configuration);
	this.resourceId = checkNotNull(resourceId);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(configuration),
		ReporterSetup.fromConfiguration(configuration));

	final RpcService metricQueryServiceRpcService = MetricUtils.startMetricsRpcService(configuration, rpcService.getAddress());
	metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture);
}
 
Example #27
Source File: YarnPreConfiguredMasterNonHaServices.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration.
 * This constructor parses the ResourceManager address from the Flink configuration and sets
 * up the HDFS access to store recovery data in the YARN application's working directory.
 *
 * @param config     The Flink configuration of this component / process.
 * @param hadoopConf The Hadoop configuration for the YARN cluster.
 *
 * @throws IOException
 *             Thrown, if the initialization of the Hadoop file system used by YARN fails.
 * @throws IllegalConfigurationException
 *             Thrown, if the Flink configuration does not properly describe the ResourceManager address and port.
 */
public YarnPreConfiguredMasterNonHaServices(
		Configuration config,
		org.apache.hadoop.conf.Configuration hadoopConf,
		HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException {

	super(config, hadoopConf);

	// track whether we successfully perform the initialization
	boolean successful = false;

	try {
		// extract the hostname and port of the resource manager
		final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS);
		final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT);

		if (rmHost == null) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing.");
		}
		if (rmPort < 0) {
			throw new IllegalConfigurationException("Config parameter '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing.");
		}
		if (rmPort <= 0 || rmPort >= 65536) {
			throw new IllegalConfigurationException("Invalid value for '" +
					YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]");
		}

		this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			ResourceManager.RESOURCE_MANAGER_NAME,
			addressResolution,
			config);

		this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
			rmHost,
			rmPort,
			Dispatcher.DISPATCHER_NAME,
			addressResolution,
			config);

		// all well!
		successful = true;
	}
	finally {
		if (!successful) {
			// quietly undo what the parent constructor initialized
			try {
				super.close();
			} catch (Throwable ignored) {}
		}
	}
}
 
Example #28
Source File: TaskManagerRunner.java    From flink with Apache License 2.0 4 votes vote down vote up
public TaskManagerRunner(Configuration configuration, PluginManager pluginManager) throws Exception {
	this.configuration = checkNotNull(configuration);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);

	this.resourceId = new ResourceID(getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort()));

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(configuration),
		ReporterSetup.fromConfiguration(configuration, pluginManager));

	final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress());
	metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	final ExternalResourceInfoProvider externalResourceInfoProvider =
		ExternalResourceUtils.createStaticExternalResourceInfoProvider(
			ExternalResourceUtils.getExternalResourceAmountMap(configuration),
			ExternalResourceUtils.externalResourceDriversFromConfig(configuration, pluginManager));

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		externalResourceInfoProvider,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture);
}
 
Example #29
Source File: TaskManagerRunner.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public TaskManagerRunner(Configuration configuration, ResourceID resourceId) throws Exception {
	this.configuration = checkNotNull(configuration);
	this.resourceId = checkNotNull(resourceId);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);
	metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, rpcService.getAddress(), LOG);

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration));

	// TODO: Temporary hack until the MetricQueryService has been ported to RpcEndpoint
	metricRegistry.startQueryService(metricQueryServiceActorSystem, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, metricQueryServiceActorSystem);
}
 
Example #30
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private HighAvailabilityServices createHighAvailabilityServices(final Configuration config) throws Exception {
	return HighAvailabilityServicesUtils.createHighAvailabilityServices(
		config,
		Executors.directExecutor(),
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
}