Java Code Examples for org.apache.flink.configuration.Configuration#keySet()

The following examples show how to use org.apache.flink.configuration.Configuration#keySet() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BootstrapTools.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 2
Source File: ContaineredTaskManagerParameters.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param taskExecutorProcessSpec The resource specifics of the task executor.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		TaskExecutorProcessSpec taskExecutorProcessSpec) {

	// obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(taskExecutorProcessSpec, envVars);
}
 
Example 3
Source File: BootstrapTools.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 4
Source File: StatefulFunctionsConfig.java    From flink-statefun with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new configuration object based on the values set in flink-conf.
 *
 * @param configuration a configuration to read the values from
 */
public StatefulFunctionsConfig(Configuration configuration) {
  StatefulFunctionsConfigValidator.validate(configuration);

  this.factoryType = configuration.get(USER_MESSAGE_SERIALIZER);
  this.flinkJobName = configuration.get(FLINK_JOB_NAME);
  this.feedbackBufferSize = configuration.get(TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING);
  this.maxAsyncOperationsPerTask = configuration.get(ASYNC_MAX_OPERATIONS_PER_TASK);

  for (String key : configuration.keySet()) {
    if (key.startsWith(MODULE_CONFIG_PREFIX)) {
      String value = configuration.get(ConfigOptions.key(key).stringType().noDefaultValue());
      String userKey = key.substring(MODULE_CONFIG_PREFIX.length());
      globalConfigurations.put(userKey, value);
    }
  }
}
 
Example 5
Source File: ClusterConfigurationInfo.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 
Example 6
Source File: BootstrapTools.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 7
Source File: ClusterConfigurationInfo.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 
Example 8
Source File: BootstrapTools.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 
Example 9
Source File: ContaineredTaskManagerParameters.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 
Example 10
Source File: ContaineredTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 
Example 11
Source File: BootstrapTools.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 
Example 12
Source File: ReporterSetup.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Set<String> findEnabledReportersInConfiguration(Configuration configuration, String includedReportersString) {
	Set<String> includedReporters = reporterListPattern.splitAsStream(includedReportersString)
		.filter(r -> !r.isEmpty()) // splitting an empty string results in an empty string on jdk9+
		.collect(Collectors.toSet());

	// use a TreeSet to make the reporter order deterministic, which is useful for testing
	Set<String> namedOrderedReporters = new TreeSet<>(String::compareTo);

	// scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine the set of enabled reporters
	for (String key : configuration.keySet()) {
		if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) {
			Matcher matcher = reporterClassPattern.matcher(key);
			if (matcher.matches()) {
				String reporterName = matcher.group(1);
				if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) {
					if (namedOrderedReporters.contains(reporterName)) {
						LOG.warn("Duplicate class configuration detected for reporter {}.", reporterName);
					} else {
						namedOrderedReporters.add(reporterName);
					}
				} else {
					LOG.info("Excluding reporter {}, not configured in reporter list ({}).", reporterName, includedReportersString);
				}
			}
		}
	}
	return namedOrderedReporters;
}
 
Example 13
Source File: BootstrapToolsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 
Example 14
Source File: InputOutputFormatContainer.java    From flink with Apache License 2.0 4 votes vote down vote up
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 
Example 15
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 
Example 16
Source File: InputOutputFormatContainer.java    From flink with Apache License 2.0 4 votes vote down vote up
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 
Example 17
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}