Java Code Examples for org.apache.flink.configuration.Configuration#getValue()

The following examples show how to use org.apache.flink.configuration.Configuration#getValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example 2
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example 3
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example 4
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example 5
Source File: StatefulFunctionsJob.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up 
public static void main(Configuration configuration) throws Exception {
  Objects.requireNonNull(configuration);

  setDefaultContextClassLoaderIfAbsent();
  setDefaultProviderIfAbsent(
      configuration, new StatefulFunctionsUniverses.ClassPathUniverseProvider());

  final StatefulFunctionsUniverse statefulFunctionsUniverse =
      StatefulFunctionsUniverses.get(
          Thread.currentThread().getContextClassLoader(), configuration);

  final StatefulFunctionsUniverseValidator statefulFunctionsUniverseValidator =
      new StatefulFunctionsUniverseValidator();
  statefulFunctionsUniverseValidator.validate(statefulFunctionsUniverse);

  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  setDefaultConfiguration(configuration, env);

  FlinkUniverse flinkUniverse = new FlinkUniverse(statefulFunctionsUniverse);
  flinkUniverse.configure(env);

  String jobName = configuration.getValue(StatefulFunctionsJobConstants.FLINK_JOB_NAME);
  env.execute(jobName);
}
Example 6
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example 7
Source File: ZooKeeperUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the configured ZooKeeper quorum (and removes whitespace, because ZooKeeper does not
 * tolerate it).
 */
public static String getZooKeeperEnsemble(Configuration flinkConf)
		throws IllegalConfigurationException {

	String zkQuorum = flinkConf.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new IllegalConfigurationException("No ZooKeeper quorum specified in config.");
	}

	// Remove all whitespace
	zkQuorum = zkQuorum.replaceAll("\\s+", "");

	return zkQuorum;
}
Example 8
Source File: ZooKeeperUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link FileSystemStateStorageHelper} instance.
 *
 * @param configuration {@link Configuration} object
 * @param prefix Prefix for the created files
 * @param <T> Type of the state objects
 * @return {@link FileSystemStateStorageHelper} instance
 * @throws IOException if file system state storage cannot be created
 */
public static <T extends Serializable> FileSystemStateStorageHelper<T> createFileSystemStateStorage(
		Configuration configuration,
		String prefix) throws IOException {

	String rootPath = configuration.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);

	if (rootPath == null || StringUtils.isBlank(rootPath)) {
		throw new IllegalConfigurationException("Missing high-availability storage path for metadata." +
				" Specify via configuration key '" + HighAvailabilityOptions.HA_STORAGE_PATH + "'.");
	} else {
		return new FileSystemStateStorageHelper<T>(rootPath, prefix);
	}
}
Example 9
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the configured ZooKeeper quorum (and removes whitespace, because ZooKeeper does not
 * tolerate it).
 */
public static String getZooKeeperEnsemble(Configuration flinkConf)
		throws IllegalConfigurationException {

	String zkQuorum = flinkConf.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new IllegalConfigurationException("No ZooKeeper quorum specified in config.");
	}

	// Remove all whitespace
	zkQuorum = zkQuorum.replaceAll("\\s+", "");

	return zkQuorum;
}
Example 10
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a {@link FileSystemStateStorageHelper} instance.
 *
 * @param configuration {@link Configuration} object
 * @param prefix Prefix for the created files
 * @param <T> Type of the state objects
 * @return {@link FileSystemStateStorageHelper} instance
 * @throws IOException if file system state storage cannot be created
 */
public static <T extends Serializable> FileSystemStateStorageHelper<T> createFileSystemStateStorage(
		Configuration configuration,
		String prefix) throws IOException {

	String rootPath = configuration.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);

	if (rootPath == null || StringUtils.isBlank(rootPath)) {
		throw new IllegalConfigurationException("Missing high-availability storage path for metadata." +
				" Specify via configuration key '" + HighAvailabilityOptions.HA_STORAGE_PATH + "'.");
	} else {
		return new FileSystemStateStorageHelper<T>(rootPath, prefix);
	}
}
Example 11
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example 12
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the configured ZooKeeper quorum (and removes whitespace, because ZooKeeper does not
 * tolerate it).
 */
public static String getZooKeeperEnsemble(Configuration flinkConf)
		throws IllegalConfigurationException {

	String zkQuorum = flinkConf.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new IllegalConfigurationException("No ZooKeeper quorum specified in config.");
	}

	// Remove all whitespace
	zkQuorum = zkQuorum.replaceAll("\\s+", "");

	return zkQuorum;
}
Example 13
Source File: ZooKeeperUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Starts a {@link CuratorFramework} instance and connects it to the given ZooKeeper
 * quorum.
 *
 * @param configuration {@link Configuration} object containing the configuration values
 * @return {@link CuratorFramework} instance
 */
public static CuratorFramework startCuratorFramework(Configuration configuration) {
	Preconditions.checkNotNull(configuration, "configuration");
	String zkQuorum = configuration.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new RuntimeException("No valid ZooKeeper quorum has been specified. " +
				"You can specify the quorum via the configuration key '" +
				HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM.key() + "'.");
	}

	int sessionTimeout = configuration.getInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT);

	int connectionTimeout = configuration.getInteger(HighAvailabilityOptions.ZOOKEEPER_CONNECTION_TIMEOUT);

	int retryWait = configuration.getInteger(HighAvailabilityOptions.ZOOKEEPER_RETRY_WAIT);

	int maxRetryAttempts = configuration.getInteger(HighAvailabilityOptions.ZOOKEEPER_MAX_RETRY_ATTEMPTS);

	String root = configuration.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_ROOT);

	String namespace = configuration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);

	boolean disableSaslClient = configuration.getBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE);

	ACLProvider aclProvider;

	ZkClientACLMode aclMode = ZkClientACLMode.fromConfig(configuration);

	if (disableSaslClient && aclMode == ZkClientACLMode.CREATOR) {
		String errorMessage = "Cannot set ACL role to " + aclMode + "  since SASL authentication is " +
				"disabled through the " + SecurityOptions.ZOOKEEPER_SASL_DISABLE.key() + " property";
		LOG.warn(errorMessage);
		throw new IllegalConfigurationException(errorMessage);
	}

	if (aclMode == ZkClientACLMode.CREATOR) {
		LOG.info("Enforcing creator for ZK connections");
		aclProvider = new SecureAclProvider();
	} else {
		LOG.info("Enforcing default ACL for ZK connections");
		aclProvider = new DefaultACLProvider();
	}

	String rootWithNamespace = generateZookeeperPath(root, namespace);

	LOG.info("Using '{}' as Zookeeper namespace.", rootWithNamespace);

	CuratorFramework cf = CuratorFrameworkFactory.builder()
			.connectString(zkQuorum)
			.sessionTimeoutMs(sessionTimeout)
			.connectionTimeoutMs(connectionTimeout)
			.retryPolicy(new ExponentialBackoffRetry(retryWait, maxRetryAttempts))
			// Curator prepends a '/' manually and throws an Exception if the
			// namespace starts with a '/'.
			.namespace(rootWithNamespace.startsWith("/") ? rootWithNamespace.substring(1) : rootWithNamespace)
			.aclProvider(aclProvider)
			.build();

	cf.start();

	return cf;
}