Java Code Examples for org.apache.flink.configuration.Configuration#keySet()

/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 

/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param taskExecutorProcessSpec The resource specifics of the task executor.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		TaskExecutorProcessSpec taskExecutorProcessSpec) {

	// obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(taskExecutorProcessSpec, envVars);
}
 

/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 

/**
 * Create a new configuration object based on the values set in flink-conf.
 *
 * @param configuration a configuration to read the values from
 */
public StatefulFunctionsConfig(Configuration configuration) {
  StatefulFunctionsConfigValidator.validate(configuration);

  this.factoryType = configuration.get(USER_MESSAGE_SERIALIZER);
  this.flinkJobName = configuration.get(FLINK_JOB_NAME);
  this.feedbackBufferSize = configuration.get(TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING);
  this.maxAsyncOperationsPerTask = configuration.get(ASYNC_MAX_OPERATIONS_PER_TASK);

  for (String key : configuration.keySet()) {
    if (key.startsWith(MODULE_CONFIG_PREFIX)) {
      String value = configuration.get(ConfigOptions.key(key).stringType().noDefaultValue());
      String userKey = key.substring(MODULE_CONFIG_PREFIX.length());
      globalConfigurations.put(userKey, value);
    }
  }
}
 

public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 

/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 

public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 

/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 

/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 

/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		long containerMemoryMB,
		int numSlots) {
	// (1) try to compute how much memory used by container
	final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);

	// (2) split the remaining Java memory between heap and off-heap
	final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config);
	// use the cut-off memory for off-heap (that was its intention)
	final long offHeapSizeMB = containerMemoryMB - heapSizeMB;

	// (3) obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(
		containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
}
 

/**
 * Writes a Flink YAML config file from a Flink Configuration object.
 * @param cfg The Flink config
 * @param file The File to write to
 * @throws IOException
 */
public static void writeConfiguration(Configuration cfg, File file) throws IOException {
	try (FileWriter fwrt = new FileWriter(file);
		PrintWriter out = new PrintWriter(fwrt)) {
		for (String key : cfg.keySet()) {
			String value = cfg.getString(key, null);
			out.print(key);
			out.print(": ");
			out.println(value);
		}
	}
}
 

private static Set<String> findEnabledReportersInConfiguration(Configuration configuration, String includedReportersString) {
	Set<String> includedReporters = reporterListPattern.splitAsStream(includedReportersString)
		.filter(r -> !r.isEmpty()) // splitting an empty string results in an empty string on jdk9+
		.collect(Collectors.toSet());

	// use a TreeSet to make the reporter order deterministic, which is useful for testing
	Set<String> namedOrderedReporters = new TreeSet<>(String::compareTo);

	// scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine the set of enabled reporters
	for (String key : configuration.keySet()) {
		if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) {
			Matcher matcher = reporterClassPattern.matcher(key);
			if (matcher.matches()) {
				String reporterName = matcher.group(1);
				if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) {
					if (namedOrderedReporters.contains(reporterName)) {
						LOG.warn("Duplicate class configuration detected for reporter {}.", reporterName);
					} else {
						namedOrderedReporters.add(reporterName);
					}
				} else {
					LOG.info("Excluding reporter {}, not configured in reporter list ({}).", reporterName, includedReportersString);
				}
			}
		}
	}
	return namedOrderedReporters;
}
 

@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 

public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 

@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}
 

public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) {
	for (String key : parameters.keySet()) {
		addParameters(operatorId, key, parameters.getString(key, null));
	}
	return this;
}
 

@Test
public void testSubstituteConfigKeyPrefix() {
	String deprecatedPrefix1 = "deprecated-prefix";
	String deprecatedPrefix2 = "-prefix-2";
	String deprecatedPrefix3 = "prefix-3";

	String designatedPrefix1 = "p1";
	String designatedPrefix2 = "ppp";
	String designatedPrefix3 = "zzz";

	String depr1 = deprecatedPrefix1 + "var";
	String depr2 = deprecatedPrefix2 + "env";
	String depr3 = deprecatedPrefix2 + "x";

	String desig1 = designatedPrefix1 + "var";
	String desig2 = designatedPrefix2 + "env";
	String desig3 = designatedPrefix2 + "x";

	String val1 = "1";
	String val2 = "2";
	String val3Depr = "3-";
	String val3Desig = "3+";

	// config contains only deprecated key 1, and for key 2 both deprecated and designated
	Configuration cfg = new Configuration();
	cfg.setString(depr1, val1);
	cfg.setString(depr2, val2);
	cfg.setString(depr3, val3Depr);
	cfg.setString(desig3, val3Desig);

	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2);
	BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3);

	assertEquals(val1, cfg.getString(desig1, null));
	assertEquals(val2, cfg.getString(desig2, null));
	assertEquals(val3Desig, cfg.getString(desig3, null));

	// check that nothing with prefix 3 is contained
	for (String key : cfg.keySet()) {
		assertFalse(key.startsWith(designatedPrefix3));
		assertFalse(key.startsWith(deprecatedPrefix3));
	}
}