Java Code Examples for org.apache.flink.runtime.jobmanager.HighAvailabilityMode#isHighAvailabilityModeActivated()

/**
 * For non-HA cluster, {@link JobManagerOptions#ADDRESS} has be set to Kubernetes service name on client side. See
 * {@link KubernetesClusterDescriptor#deployClusterInternal}. So the TaskManager will use service address to contact
 * with JobManager.
 * For HA cluster, {@link JobManagerOptions#ADDRESS} will be set to the pod ip address. The TaskManager use Zookeeper
 * or other high-availability service to find the address of JobManager.
 *
 * @return Updated configuration
 */
static Configuration loadConfiguration() {
	final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
	Preconditions.checkNotNull(
		configDir,
		"Flink configuration directory (%s) in environment should not be null!",
		ConfigConstants.ENV_FLINK_CONF_DIR);

	final Configuration configuration = GlobalConfiguration.loadConfiguration(configDir);

	if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) {
		final String ipAddress = System.getenv().get(Constants.ENV_FLINK_POD_IP_ADDRESS);
		Preconditions.checkState(
			ipAddress != null,
			"JobManager ip address environment variable %s not set",
			Constants.ENV_FLINK_POD_IP_ADDRESS);
		configuration.setString(JobManagerOptions.ADDRESS, ipAddress);
		configuration.setString(RestOptions.ADDRESS, ipAddress);
	}

	return configuration;
}
 

@VisibleForTesting
CompletableFuture<Void> fixJobIdAndRunApplicationAsync(
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor) {

	final Optional<String> configuredJobId =
			configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);

	if (!HighAvailabilityMode.isHighAvailabilityModeActivated(configuration) && !configuredJobId.isPresent()) {
		return runApplicationAsync(dispatcher, scheduledExecutor, false);
	}

	if (!configuredJobId.isPresent()) {
		configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, ZERO_JOB_ID.toHexString());
	}
	return runApplicationAsync(dispatcher, scheduledExecutor, true);
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 

public boolean isInternalServiceEnabled() {
	return !HighAvailabilityMode.isHighAvailabilityModeActivated(flinkConfig);
}
 

private ClusterClientProvider<String> deployClusterInternal(
		String entryPoint,
		ClusterSpecification clusterSpecification,
		boolean detached) throws ClusterDeploymentException {
	final ClusterEntrypoint.ExecutionMode executionMode = detached ?
		ClusterEntrypoint.ExecutionMode.DETACHED
		: ClusterEntrypoint.ExecutionMode.NORMAL;
	flinkConfig.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString());

	flinkConfig.setString(KubernetesConfigOptionsInternal.ENTRY_POINT_CLASS, entryPoint);

	// Rpc, blob, rest, taskManagerRpc ports need to be exposed, so update them to fixed values.
	KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, BlobServerOptions.PORT, Constants.BLOB_SERVER_PORT);
	KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, TaskManagerOptions.RPC_PORT, Constants.TASK_MANAGER_RPC_PORT);
	KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, RestOptions.BIND_PORT, Constants.REST_PORT);

	if (HighAvailabilityMode.isHighAvailabilityModeActivated(flinkConfig)) {
		flinkConfig.setString(HighAvailabilityOptions.HA_CLUSTER_ID, clusterId);
		KubernetesUtils.checkAndUpdatePortConfigOption(
			flinkConfig,
			HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE,
			flinkConfig.get(JobManagerOptions.PORT));
	}

	try {
		final KubernetesJobManagerParameters kubernetesJobManagerParameters =
			new KubernetesJobManagerParameters(flinkConfig, clusterSpecification);

		final KubernetesJobManagerSpecification kubernetesJobManagerSpec =
			KubernetesJobManagerFactory.buildKubernetesJobManagerSpecification(kubernetesJobManagerParameters);

		client.createJobManagerComponent(kubernetesJobManagerSpec);

		return createClusterClientProvider(clusterId);
	} catch (Exception e) {
		try {
			LOG.warn("Failed to create the Kubernetes cluster \"{}\", try to clean up the residual resources.", clusterId);
			client.stopAndCleanupCluster(clusterId);
		} catch (Exception e1) {
			LOG.info("Failed to stop and clean up the Kubernetes cluster \"{}\".", clusterId, e1);
		}
		throw new ClusterDeploymentException("Could not create Kubernetes cluster \"" + clusterId + "\".", e);
	}
}
 

/**
 * Creates a BlobStore based on the parameters set in the configuration.
 *
 * @param config
 * 		configuration to use
 *
 * @return a (distributed) blob store for high availability
 *
 * @throws IOException
 * 		thrown if the (distributed) file storage cannot be created
 */
public static BlobStoreService createBlobStoreFromConfig(Configuration config) throws IOException {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(config)) {
		return createFileSystemBlobStore(config);
	} else {
		return new VoidBlobStore();
	}
}
 

/**
 * Creates a BlobStore based on the parameters set in the configuration.
 *
 * @param config
 * 		configuration to use
 *
 * @return a (distributed) blob store for high availability
 *
 * @throws IOException
 * 		thrown if the (distributed) file storage cannot be created
 */
public static BlobStoreService createBlobStoreFromConfig(Configuration config) throws IOException {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(config)) {
		return createFileSystemBlobStore(config);
	} else {
		return new VoidBlobStore();
	}
}
 

/**
 * Creates a BlobStore based on the parameters set in the configuration.
 *
 * @param config
 * 		configuration to use
 *
 * @return a (distributed) blob store for high availability
 *
 * @throws IOException
 * 		thrown if the (distributed) file storage cannot be created
 */
public static BlobStoreService createBlobStoreFromConfig(Configuration config) throws IOException {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(config)) {
		return createFileSystemBlobStore(config);
	} else {
		return new VoidBlobStore();
	}
}