Java Code Examples for org.apache.flink.runtime.jobmanager.HighAvailabilityMode

The following examples show how to use org.apache.flink.runtime.jobmanager.HighAvailabilityMode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 2
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 3
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 4
Source Project: flink   Source File: YarnHighAvailabilityServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 5
Source Project: flink   Source File: YarnHighAvailabilityServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 6
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 7
Source Project: flink   Source File: KubernetesEntrypointUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * For non-HA cluster, {@link JobManagerOptions#ADDRESS} has be set to Kubernetes service name on client side. See
 * {@link KubernetesClusterDescriptor#deployClusterInternal}. So the TaskManager will use service address to contact
 * with JobManager.
 * For HA cluster, {@link JobManagerOptions#ADDRESS} will be set to the pod ip address. The TaskManager use Zookeeper
 * or other high-availability service to find the address of JobManager.
 *
 * @return Updated configuration
 */
static Configuration loadConfiguration() {
	final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
	Preconditions.checkNotNull(
		configDir,
		"Flink configuration directory (%s) in environment should not be null!",
		ConfigConstants.ENV_FLINK_CONF_DIR);

	final Configuration configuration = GlobalConfiguration.loadConfiguration(configDir);

	if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) {
		final String ipAddress = System.getenv().get(Constants.ENV_FLINK_POD_IP_ADDRESS);
		Preconditions.checkState(
			ipAddress != null,
			"JobManager ip address environment variable %s not set",
			Constants.ENV_FLINK_POD_IP_ADDRESS);
		configuration.setString(JobManagerOptions.ADDRESS, ipAddress);
		configuration.setString(RestOptions.ADDRESS, ipAddress);
	}

	return configuration;
}
 
Example 8
Source Project: flink   Source File: KubernetesClusterDescriptorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDeployHighAvailabilitySessionCluster() throws ClusterDeploymentException {
	flinkConfig.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.toString());
	final ClusterClient<String> clusterClient = deploySessionCluster().getClusterClient();
	checkClusterClient(clusterClient);

	final Container jmContainer = kubeClient
		.apps()
		.deployments()
		.list()
		.getItems()
		.get(0)
		.getSpec()
		.getTemplate()
		.getSpec()
		.getContainers()
		.get(0);
	assertTrue(
		"Environment " + ENV_FLINK_POD_IP_ADDRESS + " should be set.",
		jmContainer.getEnv().stream()
			.map(EnvVar::getName)
			.collect(Collectors.toList())
			.contains(ENV_FLINK_POD_IP_ADDRESS));

	clusterClient.close();
}
 
Example 9
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 10
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static ClientHighAvailabilityServices createClientHAService(Configuration configuration) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final String webMonitorAddress = getWebMonitorAddress(configuration, AddressResolution.TRY_ADDRESS_RESOLUTION);
			return new StandaloneClientHAServices(webMonitorAddress);
		case ZOOKEEPER:
			final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
			return new ZooKeeperClientHAServices(client, configuration);
		case FACTORY_CLASS:
			return createCustomClientHAServices(configuration);
		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 11
Source Project: flink   Source File: ApplicationDispatcherBootstrap.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
CompletableFuture<Void> fixJobIdAndRunApplicationAsync(
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor) {

	final Optional<String> configuredJobId =
			configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);

	if (!HighAvailabilityMode.isHighAvailabilityModeActivated(configuration) && !configuredJobId.isPresent()) {
		return runApplicationAsync(dispatcher, scheduledExecutor, false);
	}

	if (!configuredJobId.isPresent()) {
		configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, ZERO_JOB_ID.toHexString());
	}
	return runApplicationAsync(dispatcher, scheduledExecutor, true);
}
 
Example 12
Source Project: flink   Source File: ApplicationDispatcherBootstrapTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobIdDefaultsToZeroWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 0L)));
}
 
Example 13
Source Project: flink   Source File: ApplicationDispatcherBootstrapTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStaticJobIdWithHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 2L)));
}
 
Example 14
Source Project: Flink-CEPplus   Source File: MesosServicesUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 15
@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 
Example 16
@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 
Example 17
Source Project: flink   Source File: MesosServicesUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 18
Source Project: flink   Source File: StandaloneJobClusterEntryPoint.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 
Example 19
Source Project: flink   Source File: HighAvailabilityServicesUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 
Example 20
@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 
Example 21
@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 
Example 22
Source Project: flink   Source File: InternalServiceDecoratorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDisableInternalService() throws IOException {
	this.flinkConfig.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final List<HasMetadata> resources = this.internalServiceDecorator.buildAccompanyingKubernetesResources();
	assertEquals(0, resources.size());
}
 
Example 23
Source Project: flink   Source File: MesosServicesUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 24
Source Project: flink   Source File: HighAvailabilityServicesUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Executors.directExecutor();

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 
Example 25
Source Project: flink   Source File: ApplicationDispatcherBootstrapTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOnlyOneJobIsAllowedWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 
Example 26
Source Project: flink   Source File: ApplicationDispatcherBootstrapTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOnlyOneJobAllowedWithStaticJobIdAndHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 
Example 27
Source Project: Flink-CEPplus   Source File: ZooKeeperUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Returns whether {@link HighAvailabilityMode#ZOOKEEPER} is configured.
 */
public static boolean isZooKeeperRecoveryMode(Configuration flinkConf) {
	return HighAvailabilityMode.fromConfig(flinkConf).equals(HighAvailabilityMode.ZOOKEEPER);
}
 
Example 28
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 29
Source Project: flink   Source File: ZooKeeperUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Returns whether {@link HighAvailabilityMode#ZOOKEEPER} is configured.
 */
public static boolean isZooKeeperRecoveryMode(Configuration flinkConf) {
	return HighAvailabilityMode.fromConfig(flinkConf).equals(HighAvailabilityMode.ZOOKEEPER);
}
 
Example 30
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}