Java Code Examples for org.apache.flink.configuration.Configuration#keySet()
The following examples show how to use
org.apache.flink.configuration.Configuration#keySet() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BootstrapTools.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Sets the value of a new config key to the value of a deprecated config key. Taking into * account the changed prefix. * @param config Config to write * @param deprecatedPrefix Old prefix of key * @param designatedPrefix New prefix of key */ public static void substituteDeprecatedConfigPrefix( Configuration config, String deprecatedPrefix, String designatedPrefix) { // set the designated key only if it is not set already final int prefixLen = deprecatedPrefix.length(); Configuration replacement = new Configuration(); for (String key : config.keySet()) { if (key.startsWith(deprecatedPrefix)) { String newKey = designatedPrefix + key.substring(prefixLen); if (!config.containsKey(newKey)) { replacement.setString(newKey, config.getString(key, null)); } } } config.addAll(replacement); }
Example 2
Source File: ContaineredTaskManagerParameters.java From flink with Apache License 2.0 | 6 votes |
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param taskExecutorProcessSpec The resource specifics of the task executor. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, TaskExecutorProcessSpec taskExecutorProcessSpec) { // obtain the additional environment variables from the configuration final HashMap<String, String> envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix String envVarKey = key.substring(prefix.length()); envVars.put(envVarKey, config.getString(key, null)); } } // done return new ContaineredTaskManagerParameters(taskExecutorProcessSpec, envVars); }
Example 3
Source File: BootstrapTools.java From flink with Apache License 2.0 | 6 votes |
/** * Sets the value of a new config key to the value of a deprecated config key. Taking into * account the changed prefix. * @param config Config to write * @param deprecatedPrefix Old prefix of key * @param designatedPrefix New prefix of key */ public static void substituteDeprecatedConfigPrefix( Configuration config, String deprecatedPrefix, String designatedPrefix) { // set the designated key only if it is not set already final int prefixLen = deprecatedPrefix.length(); Configuration replacement = new Configuration(); for (String key : config.keySet()) { if (key.startsWith(deprecatedPrefix)) { String newKey = designatedPrefix + key.substring(prefixLen); if (!config.containsKey(newKey)) { replacement.setString(newKey, config.getString(key, null)); } } } config.addAll(replacement); }
Example 4
Source File: StatefulFunctionsConfig.java From flink-statefun with Apache License 2.0 | 6 votes |
/** * Create a new configuration object based on the values set in flink-conf. * * @param configuration a configuration to read the values from */ public StatefulFunctionsConfig(Configuration configuration) { StatefulFunctionsConfigValidator.validate(configuration); this.factoryType = configuration.get(USER_MESSAGE_SERIALIZER); this.flinkJobName = configuration.get(FLINK_JOB_NAME); this.feedbackBufferSize = configuration.get(TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING); this.maxAsyncOperationsPerTask = configuration.get(ASYNC_MAX_OPERATIONS_PER_TASK); for (String key : configuration.keySet()) { if (key.startsWith(MODULE_CONFIG_PREFIX)) { String value = configuration.get(ConfigOptions.key(key).stringType().noDefaultValue()); String userKey = key.substring(MODULE_CONFIG_PREFIX.length()); globalConfigurations.put(userKey, value); } } }
Example 5
Source File: ClusterConfigurationInfo.java From flink with Apache License 2.0 | 6 votes |
public static ClusterConfigurationInfo from(Configuration config) { ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size()); for (String key : config.keySet()) { String value = config.getString(key, null); // Mask key values which contain sensitive information if (value != null && GlobalConfiguration.isSensitive(key)) { value = GlobalConfiguration.HIDDEN_CONTENT; } clusterConfig.add(new ClusterConfigurationInfoEntry(key, value)); } return clusterConfig; }
Example 6
Source File: BootstrapTools.java From flink with Apache License 2.0 | 6 votes |
/** * Sets the value of a new config key to the value of a deprecated config key. Taking into * account the changed prefix. * @param config Config to write * @param deprecatedPrefix Old prefix of key * @param designatedPrefix New prefix of key */ public static void substituteDeprecatedConfigPrefix( Configuration config, String deprecatedPrefix, String designatedPrefix) { // set the designated key only if it is not set already final int prefixLen = deprecatedPrefix.length(); Configuration replacement = new Configuration(); for (String key : config.keySet()) { if (key.startsWith(deprecatedPrefix)) { String newKey = designatedPrefix + key.substring(prefixLen); if (!config.containsKey(newKey)) { replacement.setString(newKey, config.getString(key, null)); } } } config.addAll(replacement); }
Example 7
Source File: ClusterConfigurationInfo.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static ClusterConfigurationInfo from(Configuration config) { ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size()); for (String key : config.keySet()) { String value = config.getString(key, null); // Mask key values which contain sensitive information if (value != null && GlobalConfiguration.isSensitive(key)) { value = GlobalConfiguration.HIDDEN_CONTENT; } clusterConfig.add(new ClusterConfigurationInfoEntry(key, value)); } return clusterConfig; }
Example 8
Source File: BootstrapTools.java From flink with Apache License 2.0 | 5 votes |
/** * Writes a Flink YAML config file from a Flink Configuration object. * @param cfg The Flink config * @param file The File to write to * @throws IOException */ public static void writeConfiguration(Configuration cfg, File file) throws IOException { try (FileWriter fwrt = new FileWriter(file); PrintWriter out = new PrintWriter(fwrt)) { for (String key : cfg.keySet()) { String value = cfg.getString(key, null); out.print(key); out.print(": "); out.println(value); } } }
Example 9
Source File: ContaineredTaskManagerParameters.java From flink with Apache License 2.0 | 5 votes |
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, long containerMemoryMB, int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; // (3) obtain the additional environment variables from the configuration final HashMap<String, String> envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix String envVarKey = key.substring(prefix.length()); envVars.put(envVarKey, config.getString(key, null)); } } // done return new ContaineredTaskManagerParameters( containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars); }
Example 10
Source File: ContaineredTaskManagerParameters.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, long containerMemoryMB, int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; // (3) obtain the additional environment variables from the configuration final HashMap<String, String> envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix String envVarKey = key.substring(prefix.length()); envVars.put(envVarKey, config.getString(key, null)); } } // done return new ContaineredTaskManagerParameters( containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars); }
Example 11
Source File: BootstrapTools.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Writes a Flink YAML config file from a Flink Configuration object. * @param cfg The Flink config * @param file The File to write to * @throws IOException */ public static void writeConfiguration(Configuration cfg, File file) throws IOException { try (FileWriter fwrt = new FileWriter(file); PrintWriter out = new PrintWriter(fwrt)) { for (String key : cfg.keySet()) { String value = cfg.getString(key, null); out.print(key); out.print(": "); out.println(value); } } }
Example 12
Source File: ReporterSetup.java From flink with Apache License 2.0 | 5 votes |
private static Set<String> findEnabledReportersInConfiguration(Configuration configuration, String includedReportersString) { Set<String> includedReporters = reporterListPattern.splitAsStream(includedReportersString) .filter(r -> !r.isEmpty()) // splitting an empty string results in an empty string on jdk9+ .collect(Collectors.toSet()); // use a TreeSet to make the reporter order deterministic, which is useful for testing Set<String> namedOrderedReporters = new TreeSet<>(String::compareTo); // scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine the set of enabled reporters for (String key : configuration.keySet()) { if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) { Matcher matcher = reporterClassPattern.matcher(key); if (matcher.matches()) { String reporterName = matcher.group(1); if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) { if (namedOrderedReporters.contains(reporterName)) { LOG.warn("Duplicate class configuration detected for reporter {}.", reporterName); } else { namedOrderedReporters.add(reporterName); } } else { LOG.info("Excluding reporter {}, not configured in reporter list ({}).", reporterName, includedReportersString); } } } } return namedOrderedReporters; }
Example 13
Source File: BootstrapToolsTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testSubstituteConfigKeyPrefix() { String deprecatedPrefix1 = "deprecated-prefix"; String deprecatedPrefix2 = "-prefix-2"; String deprecatedPrefix3 = "prefix-3"; String designatedPrefix1 = "p1"; String designatedPrefix2 = "ppp"; String designatedPrefix3 = "zzz"; String depr1 = deprecatedPrefix1 + "var"; String depr2 = deprecatedPrefix2 + "env"; String depr3 = deprecatedPrefix2 + "x"; String desig1 = designatedPrefix1 + "var"; String desig2 = designatedPrefix2 + "env"; String desig3 = designatedPrefix2 + "x"; String val1 = "1"; String val2 = "2"; String val3Depr = "3-"; String val3Desig = "3+"; // config contains only deprecated key 1, and for key 2 both deprecated and designated Configuration cfg = new Configuration(); cfg.setString(depr1, val1); cfg.setString(depr2, val2); cfg.setString(depr3, val3Depr); cfg.setString(desig3, val3Desig); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3); assertEquals(val1, cfg.getString(desig1, null)); assertEquals(val2, cfg.getString(desig2, null)); assertEquals(val3Desig, cfg.getString(desig3, null)); // check that nothing with prefix 3 is contained for (String key : cfg.keySet()) { assertFalse(key.startsWith(designatedPrefix3)); assertFalse(key.startsWith(deprecatedPrefix3)); } }
Example 14
Source File: InputOutputFormatContainer.java From flink with Apache License 2.0 | 4 votes |
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) { for (String key : parameters.keySet()) { addParameters(operatorId, key, parameters.getString(key, null)); } return this; }
Example 15
Source File: BootstrapToolsTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testSubstituteConfigKeyPrefix() { String deprecatedPrefix1 = "deprecated-prefix"; String deprecatedPrefix2 = "-prefix-2"; String deprecatedPrefix3 = "prefix-3"; String designatedPrefix1 = "p1"; String designatedPrefix2 = "ppp"; String designatedPrefix3 = "zzz"; String depr1 = deprecatedPrefix1 + "var"; String depr2 = deprecatedPrefix2 + "env"; String depr3 = deprecatedPrefix2 + "x"; String desig1 = designatedPrefix1 + "var"; String desig2 = designatedPrefix2 + "env"; String desig3 = designatedPrefix2 + "x"; String val1 = "1"; String val2 = "2"; String val3Depr = "3-"; String val3Desig = "3+"; // config contains only deprecated key 1, and for key 2 both deprecated and designated Configuration cfg = new Configuration(); cfg.setString(depr1, val1); cfg.setString(depr2, val2); cfg.setString(depr3, val3Depr); cfg.setString(desig3, val3Desig); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3); assertEquals(val1, cfg.getString(desig1, null)); assertEquals(val2, cfg.getString(desig2, null)); assertEquals(val3Desig, cfg.getString(desig3, null)); // check that nothing with prefix 3 is contained for (String key : cfg.keySet()) { assertFalse(key.startsWith(designatedPrefix3)); assertFalse(key.startsWith(deprecatedPrefix3)); } }
Example 16
Source File: InputOutputFormatContainer.java From flink with Apache License 2.0 | 4 votes |
public InputOutputFormatContainer addParameters(OperatorID operatorId, Configuration parameters) { for (String key : parameters.keySet()) { addParameters(operatorId, key, parameters.getString(key, null)); } return this; }
Example 17
Source File: BootstrapToolsTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testSubstituteConfigKeyPrefix() { String deprecatedPrefix1 = "deprecated-prefix"; String deprecatedPrefix2 = "-prefix-2"; String deprecatedPrefix3 = "prefix-3"; String designatedPrefix1 = "p1"; String designatedPrefix2 = "ppp"; String designatedPrefix3 = "zzz"; String depr1 = deprecatedPrefix1 + "var"; String depr2 = deprecatedPrefix2 + "env"; String depr3 = deprecatedPrefix2 + "x"; String desig1 = designatedPrefix1 + "var"; String desig2 = designatedPrefix2 + "env"; String desig3 = designatedPrefix2 + "x"; String val1 = "1"; String val2 = "2"; String val3Depr = "3-"; String val3Desig = "3+"; // config contains only deprecated key 1, and for key 2 both deprecated and designated Configuration cfg = new Configuration(); cfg.setString(depr1, val1); cfg.setString(depr2, val2); cfg.setString(depr3, val3Depr); cfg.setString(desig3, val3Desig); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix1, designatedPrefix1); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix2, designatedPrefix2); BootstrapTools.substituteDeprecatedConfigPrefix(cfg, deprecatedPrefix3, designatedPrefix3); assertEquals(val1, cfg.getString(desig1, null)); assertEquals(val2, cfg.getString(desig2, null)); assertEquals(val3Desig, cfg.getString(desig3, null)); // check that nothing with prefix 3 is contained for (String key : cfg.keySet()) { assertFalse(key.startsWith(designatedPrefix3)); assertFalse(key.startsWith(deprecatedPrefix3)); } }