Java Code Examples for org.apache.flink.configuration.Configuration#clone()

The following examples show how to use org.apache.flink.configuration.Configuration#clone() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
}
 
Example 2
/**
 * Verifies that {@link NettyShuffleEnvironmentConfiguration#hasNewNetworkConfig(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));
}
 
Example 3
/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 
Example 4
/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 
Example 5
Source Project: flink   File: FlinkConfMountDecorator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get properties map for the cluster-side after removal of some keys.
 */
private Map<String, String> getClusterSidePropertiesMap(Configuration flinkConfig) {
	final Configuration clusterSideConfig = flinkConfig.clone();
	// Remove some configuration options that should not be taken to cluster side.
	clusterSideConfig.removeConfig(KubernetesConfigOptions.KUBE_CONFIG_FILE);
	clusterSideConfig.removeConfig(DeploymentOptionsInternal.CONF_DIR);
	return clusterSideConfig.toMap();
}