Java Code Examples for org.apache.flink.configuration.Configuration#removeConfig()

The following examples show how to use org.apache.flink.configuration.Configuration#removeConfig() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TaskManagerHeapSizeCalculationJavaBashTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
Example 2
Source File: ExecutionContext.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void setRestartStrategy(Configuration conf) {
	RestartStrategyConfiguration restartStrategy = environment.getExecution().getRestartStrategy();
	if (restartStrategy instanceof NoRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "none");
	} else if (restartStrategy instanceof FixedDelayRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
		FixedDelayRestartStrategyConfiguration fixedDelay = ((FixedDelayRestartStrategyConfiguration) restartStrategy);
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS,
				fixedDelay.getRestartAttempts());
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY,
				Duration.ofMillis(fixedDelay.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategy instanceof FailureRateRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "failure-rate");
		FailureRateRestartStrategyConfiguration failureRate = (FailureRateRestartStrategyConfiguration) restartStrategy;
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL,
				failureRate.getMaxFailureRate());
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL,
				Duration.ofMillis(failureRate.getFailureInterval().toMilliseconds()));
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_DELAY,
				Duration.ofMillis(failureRate.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategy instanceof FallbackRestartStrategyConfiguration) {
		// default is FallbackRestartStrategyConfiguration
		// see ExecutionConfig.restartStrategyConfiguration
		conf.removeConfig(RestartStrategyOptions.RESTART_STRATEGY);
	}
}
Example 3
Source File: BootstrapTools.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
Example 4
Source File: BootstrapTools.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
Example 5
Source File: NettyShuffleEnvironmentConfigurationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test for {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} with some
 * manually calculated scenarios.
 */
@Test
public void calculateHeapSizeMB() throws Exception {
	Configuration config = new Configuration();
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(64L << 20)); // 64MB
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(1L << 30)); // 1GB

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	assertEquals(900, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.2f);
	assertEquals(800, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(400, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, true);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "10m"); // 10MB
	assertEquals(890, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(390, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE); // use fraction of given memory
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10%
	assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config));
}
Example 6
Source File: FlinkConfMountDecorator.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Get properties map for the cluster-side after removal of some keys.
 */
private Map<String, String> getClusterSidePropertiesMap(Configuration flinkConfig) {
	final Configuration clusterSideConfig = flinkConfig.clone();
	// Remove some configuration options that should not be taken to cluster side.
	clusterSideConfig.removeConfig(KubernetesConfigOptions.KUBE_CONFIG_FILE);
	clusterSideConfig.removeConfig(DeploymentOptionsInternal.CONF_DIR);
	return clusterSideConfig.toMap();
}
Example 7
Source File: BootstrapTools.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
Example 8
Source File: ActiveResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
private Configuration createActiveResourceManagerConfiguration(Configuration originalConfiguration) {
	final Configuration copiedConfig = new Configuration(originalConfiguration);
	// In active mode, it's depend on the ResourceManager to set the ResourceID of TaskManagers.
	copiedConfig.removeConfig(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID);
	return TaskExecutorProcessUtils.getConfigurationMapLegacyTaskManagerHeapSizeToConfigOption(
		copiedConfig, TaskManagerOptions.TOTAL_PROCESS_MEMORY);
}
Example 9
Source File: StandaloneResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Get the configuration for standalone ResourceManager, overwrite invalid configs.
 *
 * @param configuration configuration object
 * @return the configuration for standalone ResourceManager
 */
private static Configuration getConfigurationWithoutMaxSlotNumberIfSet(Configuration configuration) {
	final Configuration copiedConfig = new Configuration(configuration);
	// The max slot limit should not take effect for standalone cluster, we overwrite the configure in case user
	// sets this value by mistake.
	if (copiedConfig.removeConfig(ResourceManagerOptions.MAX_SLOT_NUM)) {
		LOG.warn("Config option {} will be ignored in standalone mode.", ResourceManagerOptions.MAX_SLOT_NUM.key());
	}
	return copiedConfig;
}