Java Code Examples for org.apache.flink.configuration.Configuration#keySet()

The following examples show how to use org.apache.flink.configuration.Configuration#keySet() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 2
public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 
Example 3
Source Project: flink   File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 4
Source Project: flink   File: ClusterConfigurationInfo.java    License: Apache License 2.0 6 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 
Example 5
/**
 * Create a new configuration object based on the values set in flink-conf.
 *
 * @param configuration a configuration to read the values from
 */
public StatefulFunctionsConfig(Configuration configuration) {
  StatefulFunctionsConfigValidator.validate(configuration);

  this.factoryType = configuration.get(USER_MESSAGE_SERIALIZER);
  this.flinkJobName = configuration.get(FLINK_JOB_NAME);
  this.feedbackBufferSize = configuration.get(TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING);
  this.maxAsyncOperationsPerTask = configuration.get(ASYNC_MAX_OPERATIONS_PER_TASK);

  for (String key : configuration.keySet()) {
    if (key.startsWith(MODULE_CONFIG_PREFIX)) {
      String value = configuration.get(ConfigOptions.key(key).stringType().noDefaultValue());
      String userKey = key.substring(MODULE_CONFIG_PREFIX.length());
      globalConfigurations.put(userKey, value);
    }
  }
}
 
Example 6
Source Project: flink   File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 7
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param taskExecutorProcessSpec The resource specifics of the task executor.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		TaskExecutorProcessSpec taskExecutorProcessSpec) {

	// obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(taskExecutorProcessSpec, envVars);
}
 
Example 8
Source Project: Flink-CEPplus   File: BootstrapTools.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 
Example 9
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 
Example 10
Source Project: flink   File: BootstrapTools.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 
Example 11
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 
Example 12
Source Project: flink   File: ReporterSetup.java    License: Apache License 2.0 5 votes vote down vote up
private static Set<String> findEnabledReportersInConfiguration(Configuration configuration, String includedReportersString) {
	Set<String> includedReporters = reporterListPattern.splitAsStream(includedReportersString)
		.filter(r -> !r.isEmpty()) // splitting an empty string results in an empty string on jdk9+
		.collect(Collectors.toSet());

	// use a TreeSet to make the reporter order deterministic, which is useful for testing
	Set<String> namedOrderedReporters = new TreeSet<>(String::compareTo);

	// scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine the set of enabled reporters
	for (String key : configuration.keySet()) {
		if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) {
			Matcher matcher = reporterClassPattern.matcher(key);
			if (matcher.matches()) {
				String reporterName = matcher.group(1);
				if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) {
					if (namedOrderedReporters.contains(reporterName)) {
						LOG.warn("Duplicate class configuration detected for reporter {}.", reporterName);
					} else {
						namedOrderedReporters.add(reporterName);
					}
				} else {
					LOG.info("Excluding reporter {}, not configured in reporter list ({}).", reporterName, includedReportersString);
				}
			}
		}
	}
	return namedOrderedReporters;
}
 
Example 13
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 
Example 14
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 
Example 15
Source Project: flink   File: BootstrapToolsTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 
Example 16
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 
Example 17
Source Project: flink   File: BootstrapToolsTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}