Java Code Examples for org.apache.flink.configuration.Configuration#setFloat()

The following examples show how to use org.apache.flink.configuration.Configuration#setFloat() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 
Example 2
/**
 * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}.
 */
@Test
public void testCalculateCutoffMB() {

	Configuration config = new Configuration();
	long containerMemoryMB = 1000L;

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f);
	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128);

	assertEquals(128,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.2f);
	assertEquals(200,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 1000);

	try {
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB);
		fail("Expected to fail with an invalid argument exception.");
	} catch (IllegalArgumentException ignored) {
		// we expected it.
	}
}
 
Example 3
/**
 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
}
 
Example 4
/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 
Example 5
/**
 * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}.
 */
@Test
public void testCalculateCutoffMB() {

	Configuration config = new Configuration();
	long containerMemoryMB = 1000L;

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f);
	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128);

	assertEquals(128,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.2f);
	assertEquals(200,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 1000);

	try {
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB);
		fail("Expected to fail with an invalid argument exception.");
	} catch (IllegalArgumentException ignored) {
		// we expected it.
	}
}
 
Example 6
/**
 * Returns a configuration for the tests.
 *
 * @param managedMemory         see {@link TaskManagerOptions#MANAGED_MEMORY_SIZE}
 * @param managedMemoryFraction see {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION}
 * @param networkBufFraction	see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION}
 * @param networkBufMin			see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN}
 * @param networkBufMax			see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}
 * @param memoryType			on-heap or off-heap
 *
 * @return configuration object
 */
private static Configuration getConfig(
	final long managedMemory,
	final float managedMemoryFraction,
	float networkBufFraction,
	long networkBufMin,
	long networkBufMax,
	MemoryType memoryType) {

	final Configuration configuration = new Configuration();

	configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), managedMemory);
	configuration.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(), managedMemoryFraction);
	configuration.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(), networkBufFraction);
	configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key(), networkBufMin);
	configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key(), networkBufMax);
	configuration.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP.key(), memoryType == MemoryType.OFF_HEAP);

	return configuration;
}
 
Example 7
/**
 * Verifies that {@link NettyShuffleEnvironmentConfiguration#hasNewNetworkConfig(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));
}
 
Example 8
Source Project: flink   File: UnalignedCheckpointITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private LocalStreamEnvironment createEnv(int parallelism, int slotsPerTaskManager, boolean slotSharing) throws IOException {
	Configuration conf = new Configuration();
	conf.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);
	conf.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, .9f);
	conf.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
			slotSharing ? (parallelism + slotsPerTaskManager - 1) / slotsPerTaskManager : parallelism * 3);

	conf.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	conf.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, temp.newFolder().toURI().toString());

	final LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(parallelism, conf);
	env.enableCheckpointing(100);
	// keep in sync with FailingMapper in #createDAG
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, Time.milliseconds(100)));
	env.getCheckpointConfig().enableUnalignedCheckpoints(true);
	return env;
}
 
Example 9
@Test
public void testConfigNetworkMemoryFraction() {
	final MemorySize networkMin = MemorySize.ZERO;
	final MemorySize networkMax = MemorySize.parse("1t");
	final float fraction = 0.2f;

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);
	conf.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, fraction);

	// validate in configurations without explicit total flink/process memory, otherwise explicit configured
	// network memory fraction might conflict with total flink/process memory minus other memory sizes
	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(taskExecutorProcessSpec.getTotalFlinkMemorySize().multiply(fraction))));
}
 
Example 10
@Test
public void testConfigJvmOverheadFraction() {
	MemorySize minSize = MemorySize.ZERO;
	MemorySize maxSize = MemorySize.parse("1t");
	@SuppressWarnings("MagicNumber") float fraction = 0.2f;

	Configuration conf = new Configuration();
	conf.set(options.getJvmOptions().getJvmOverheadMax(), maxSize);
	conf.set(options.getJvmOptions().getJvmOverheadMin(), minSize);
	conf.setFloat(options.getJvmOptions().getJvmOverheadFraction(), fraction);

	validateInAllConfigurations(
		conf,
		jobManagerProcessSpec -> assertThat(
			jobManagerProcessSpec.getJvmOverheadSize(),
			is(jobManagerProcessSpec.getTotalProcessMemorySize().multiply(fraction))));
}
 
Example 11
/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 
Example 12
/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 
Example 13
/**
 * Test for {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} with some
 * manually calculated scenarios.
 */
@Test
public void calculateHeapSizeMB() throws Exception {
	Configuration config = new Configuration();
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(64L << 20)); // 64MB
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(1L << 30)); // 1GB

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	assertEquals(900, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.2f);
	assertEquals(800, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(400, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, true);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "10m"); // 10MB
	assertEquals(890, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(390, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE); // use fraction of given memory
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10%
	assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config));
}
 
Example 14
@Test
public void testConsistencyCheckOfDerivedNetworkMemoryWithinMinMaxRangeNotMatchingFractionPasses() {
	final Configuration configuration = setupConfigWithFlinkAndTaskHeapToDeriveGivenNetworkMem(400);
	// set fraction to be extremely low to not match the derived network memory
	configuration.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, 0.001f);
	// internal validation should pass
	TaskExecutorProcessUtils.processSpecFromConfig(configuration);
}
 
Example 15
@Test
public void testConfigJvmOverheadFractionFailureNegative() {
	Configuration conf = new Configuration();
	//noinspection MagicNumber
	conf.setFloat(options.getJvmOptions().getJvmOverheadFraction(), -0.1f);
	validateFailInAllConfigurations(conf);
}
 
Example 16
@Test
public void testConfigNetworkMemoryLegacyRangeFraction() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("500m");

	final float fraction = 0.2f;

	@SuppressWarnings("deprecation")
	final ConfigOption<String> legacyOptionMin = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN;
	@SuppressWarnings("deprecation")
	final ConfigOption<String> legacyOptionMax = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX;
	@SuppressWarnings("deprecation")
	final ConfigOption<Float> legacyOptionFraction = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION;

	Configuration conf = new Configuration();
	conf.setString(legacyOptionMin, networkMin.getMebiBytes() + "m");
	conf.setString(legacyOptionMax, networkMax.getMebiBytes() + "m");

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> {
		assertThat(taskExecutorProcessSpec.getNetworkMemSize().getBytes(), greaterThanOrEqualTo(networkMin.getBytes()));
		assertThat(taskExecutorProcessSpec.getNetworkMemSize().getBytes(), lessThanOrEqualTo(networkMax.getBytes()));
	});

	conf.setString(legacyOptionMin, "0m");
	conf.setString(legacyOptionMax, "1t");
	conf.setFloat(legacyOptionFraction, fraction);

	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(taskExecutorProcessSpec.getTotalFlinkMemorySize().multiply(fraction))));
}
 
Example 17
@Test
public void testConfigManagedMemoryFraction() {
	final float fraction = 0.5f;

	Configuration conf = new Configuration();
	conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, fraction);

	// managed memory fraction is only used when managed memory size is not explicitly configured
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(taskExecutorProcessSpec.getTotalFlinkMemorySize().multiply(fraction))));
}
 
Example 18
@Test
public void testConfigManagedMemoryFractionFailure() {
	final Configuration conf = new Configuration();
	conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, -0.1f);
	validateFailInConfigurationsWithoutExplicitManagedMem(conf);

	conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 1.0f);
	validateFailInConfigurationsWithoutExplicitManagedMem(conf);
}
 
Example 19
@Test
public void testFlinkInternalMemoryFractionAddUpFailure() {
	final float networkFraction = 0.6f;
	final float managedFraction = 0.6f;

	Configuration conf = new Configuration();
	conf.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, networkFraction);
	conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedFraction);

	// if managed memory size is explicitly configured, then managed memory fraction will be ignored
	validateFailInConfigWithExplicitTotalFlinkMem(conf);
	validateFailInConfigWithExplicitTotalProcessMem(conf);
}
 
Example 20
@Test
public void testConfigJvmOverheadFractionFailureNoLessThanOne() {
	Configuration conf = new Configuration();
	conf.setFloat(options.getJvmOptions().getJvmOverheadFraction(), 1.0f);
	validateFailInAllConfigurations(conf);
}