Java Code Examples for org.apache.flink.configuration.Configuration#getFloat()

The following examples show how to use org.apache.flink.configuration.Configuration#getFloat() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * This tests that per default the off heap memory is set to what the network buffers require.
 */
@Test
public void testOffHeapMemoryWithDefaultConfiguration() {
	Configuration conf = new Configuration();

	ContaineredTaskManagerParameters params =
		ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);

	final float memoryCutoffRatio = conf.getFloat(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_RATIO,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF_RATIO);
	final int minCutoff = conf.getInteger(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_MIN,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF);

	long cutoff = Math.max((long) (CONTAINER_MEMORY * memoryCutoffRatio), minCutoff);
	final long networkBufMB =
		calculateNetworkBufferMemory(
			(CONTAINER_MEMORY - cutoff) << 20, // megabytes to bytes
			conf) >> 20; // bytes to megabytes
	assertEquals(networkBufMB + cutoff, params.taskManagerDirectMemoryLimitMB());
}
 
Example 2
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalProcessMemory overall available memory to use (in bytes)
 * @param config configuration object
 *
 * @return memory to use for network buffers (in bytes)
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalProcessMemory, Configuration config) {
	final int segmentSize = ConfigurationParserUtils.getPageSize(config);

	final long networkReservedMemory;
	if (hasNewNetworkConfig(config)) {
		float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkMemoryByFraction = (long) (totalProcessMemory * networkBufFraction);
		networkReservedMemory = calculateNewNetworkBufferMemory(config, networkMemoryByFraction, totalProcessMemory);
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS);
		networkReservedMemory = (long) numNetworkBuffers * (long) segmentSize;

		checkOldNetworkConfig(numNetworkBuffers);

		ConfigurationParserUtils.checkConfigParameter(networkReservedMemory < totalProcessMemory,
			networkReservedMemory, NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key(),
			"Network buffer memory size too large: " + networkReservedMemory + " >= " +
				totalProcessMemory + " (total JVM memory size)");
	}

	return networkReservedMemory;
}
 
Example 3
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 
Example 4
Source Project: flink   File: TaskManagerServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets the size of managed memory from the heap size after subtracting network buffer memory.
 * All values are in bytes.
 */
public static long getManagedMemoryFromHeapAndManaged(Configuration config, long heapAndManagedMemory) {
	if (config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		// take the configured absolute value
		final String sizeValue = config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE);
		try {
			return MemorySize.parse(sizeValue, MEGA_BYTES).getBytes();
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	}
	else {
		// calculate managed memory size via fraction
		final float fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
		return (long) (fraction * heapAndManagedMemory);
	}
}
 
Example 5
Source Project: flink   File: ProcessMemoryUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static RangeFraction getRangeFraction(
		MemorySize minSize,
		MemorySize maxSize,
		ConfigOption<Float> fractionOption,
		Configuration config) {
	double fraction = config.getFloat(fractionOption);
	try {
		return new RangeFraction(minSize, maxSize, fraction);
	} catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException(
			String.format(
				"Inconsistently configured %s (%s) and its min (%s), max (%s) value",
				fractionOption,
				fraction,
				minSize.toHumanReadableString(),
				maxSize.toHumanReadableString()),
			e);
	}
}
 
Example 6
/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 
Example 7
Source Project: Flink-CEPplus   File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 
Example 8
/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 
Example 9
/**
 * Calculates the amount of memory used for network buffers inside the current JVM instance
 * based on the available heap or the max heap size and the according configuration parameters.
 *
 * <p>For containers or when started via scripts, if started with a memory limit and set to use
 * off-heap memory, the maximum heap size for the JVM is adjusted accordingly and we are able
 * to extract the intended values from this.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link TaskManagerOptions#MANAGED_MEMORY_SIZE},</li>
 *  <li>{@link TaskManagerOptions#MANAGED_MEMORY_FRACTION},</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param config configuration object
 * @param maxJvmHeapMemory the maximum JVM heap size (in bytes)
 *
 * @return memory to use for network buffers (in bytes)
 */
@VisibleForTesting
public static long calculateNewNetworkBufferMemory(Configuration config, long maxJvmHeapMemory) {
	// The maximum heap memory has been adjusted as in TaskManagerServices#calculateHeapSizeMB
	// and we need to invert these calculations.
	final long heapAndManagedMemory;
	final MemoryType memoryType = ConfigurationParserUtils.getMemoryType(config);
	if (memoryType == MemoryType.HEAP) {
		heapAndManagedMemory = maxJvmHeapMemory;
	} else if (memoryType == MemoryType.OFF_HEAP) {
		long configuredMemory = ConfigurationParserUtils.getManagedMemorySize(config) << 20; // megabytes to bytes
		if (configuredMemory > 0) {
			// The maximum heap memory has been adjusted according to configuredMemory, i.e.
			// maxJvmHeap = jvmHeapNoNet - configuredMemory
			heapAndManagedMemory = maxJvmHeapMemory + configuredMemory;
		} else {
			// The maximum heap memory has been adjusted according to the fraction, i.e.
			// maxJvmHeap = jvmHeapNoNet - jvmHeapNoNet * managedFraction = jvmHeapNoNet * (1 - managedFraction)
			heapAndManagedMemory = (long) (maxJvmHeapMemory / (1.0 - ConfigurationParserUtils.getManagedMemoryFraction(config)));
		}
	} else {
		throw new RuntimeException("No supported memory type detected.");
	}

	// finally extract the network buffer memory size again from:
	// heapAndManagedMemory = totalProcessMemory - networkReservedMemory
	//                      = totalProcessMemory - Math.min(networkBufMax, Math.max(networkBufMin, totalProcessMemory * networkBufFraction)
	// totalProcessMemory = heapAndManagedMemory / (1.0 - networkBufFraction)
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);

	// Converts to double for higher precision. Converting via string achieve higher precision for those
	// numbers can not be represented preciously by float, like 0.4f.
	double heapAndManagedFraction = 1.0 - Double.valueOf(Float.toString(networkBufFraction));
	long totalProcessMemory = (long) (heapAndManagedMemory / heapAndManagedFraction);
	long networkMemoryByFraction = (long) (totalProcessMemory * networkBufFraction);

	// Do not need to check the maximum allowed memory since the computed total memory should always
	// be larger than the computed network buffer memory as long as the fraction is less than 1.
	return calculateNewNetworkBufferMemory(config, networkMemoryByFraction, Long.MAX_VALUE);
}
 
Example 10
Source Project: flink   File: ConfigurationParserUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parses the configuration to get the fraction of managed memory and validates the value.
 *
 * @param configuration configuration object
 * @return fraction of managed memory
 */
public static float getManagedMemoryFraction(Configuration configuration) {
	float managedMemoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);

	checkConfigParameter(managedMemoryFraction > 0.0f && managedMemoryFraction < 1.0f, managedMemoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	return managedMemoryFraction;
}
 
Example 11
/**
 * This tests that per default the off heap memory is set to what the network buffers require.
 */
@Test
public void testOffHeapMemoryWithDefaultConfiguration() {
	Configuration conf = new Configuration();

	ContaineredTaskManagerParameters params =
		ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);

	final float memoryCutoffRatio = conf.getFloat(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_RATIO,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF_RATIO);
	final int minCutoff = conf.getInteger(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_MIN,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF);

	long cutoff = Math.max((long) (CONTAINER_MEMORY * memoryCutoffRatio), minCutoff);
	final long networkBufMB = TaskManagerServices.getReservedNetworkMemory(
		conf,
		(CONTAINER_MEMORY - cutoff) << 20 // megabytes to bytes
	) >> 20; // bytes to megabytes

	// this is unfortunately necessary due to rounding errors that happen due to back and
	// forth conversion between bytes and megabytes
	// ideally all logic calculates precisely with bytes and we use coarser units only un
	// user-facing configuration and parametrization classes
	assertThat(networkBufMB + cutoff,
		anyOf(
			equalTo(params.taskManagerDirectMemoryLimitMB()),
			equalTo(params.taskManagerDirectMemoryLimitMB() - 1)));
}
 
Example 12
Source Project: flink   File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 
Example 13
Source Project: beam   File: FlinkExecutionEnvironments.java    License: Apache License 2.0 5 votes vote down vote up
private static void setManagedMemoryByFraction(final Configuration config) {
  if (!config.containsKey("taskmanager.memory.managed.size")) {
    float managedMemoryFraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
    long freeHeapMemory = EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag();
    long managedMemorySize = (long) (freeHeapMemory * managedMemoryFraction);
    config.setString("taskmanager.memory.managed.size", String.valueOf(managedMemorySize));
  }
}
 
Example 14
Source Project: flink   File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 
Example 15
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link TaskManagerOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalJavaMemorySize
 * 		overall available memory to use (heap and off-heap, in bytes)
 * @param config
 * 		configuration object
 *
 * @return memory to use for network buffers (in bytes); at least one memory segment
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalJavaMemorySize, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySize > 0);

	int segmentSize =
		checkedDownCast(MemorySize.parse(config.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes());

	final long networkBufBytes;
	if (TaskManagerServicesConfiguration.hasNewNetworkBufConf(config)) {
		// new configuration based on fractions of available memory with selectable min and max
		float networkBufFraction = config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkBufMin = MemorySize.parse(config.getString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
		long networkBufMax = MemorySize.parse(config.getString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

		TaskManagerServicesConfiguration
			.checkNetworkBufferConfig(segmentSize, networkBufFraction, networkBufMin, networkBufMax);

		networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
			(long) (networkBufFraction * totalJavaMemorySize)));

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
		networkBufBytes = (long) numNetworkBuffers * (long) segmentSize;

		TaskManagerServicesConfiguration.checkNetworkConfigOld(numNetworkBuffers);

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	}

	return networkBufBytes;
}
 
Example 16
/**
 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
 * based on the total memory to use and the given configuration parameters.
 *
 * @param totalJavaMemorySizeMB
 * 		overall available memory to use (heap and off-heap)
 * @param config
 * 		configuration object
 *
 * @return heap memory to use (in megabytes)
 */
public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySizeMB > 0);

	// subtract the Java memory used for network buffers (always off-heap)
	final long networkBufMB =
		calculateNetworkBufferMemory(
			totalJavaMemorySizeMB << 20, // megabytes to bytes
			config) >> 20; // bytes to megabytes
	final long remainingJavaMemorySizeMB = totalJavaMemorySizeMB - networkBufMB;

	// split the available Java memory between heap and off-heap

	final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);

	final long heapSizeMB;
	if (useOffHeap) {

		long offHeapSize;
		String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
		if (!config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
			try {
				offHeapSize = MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
			} catch (IllegalArgumentException e) {
				throw new IllegalConfigurationException(
					"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
			}
		} else {
			offHeapSize = Long.valueOf(managedMemorySizeDefaultVal);
		}

		if (offHeapSize <= 0) {
			// calculate off-heap section via fraction
			double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
			offHeapSize = (long) (fraction * remainingJavaMemorySizeMB);
		}

		TaskManagerServicesConfiguration
			.checkConfigParameter(offHeapSize < remainingJavaMemorySizeMB, offHeapSize,
				TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
				"Managed memory size too large for " + networkBufMB +
					" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
					" MB JVM memory");

		heapSizeMB = remainingJavaMemorySizeMB - offHeapSize;
	} else {
		heapSizeMB = remainingJavaMemorySizeMB;
	}

	return heapSizeMB;
}
 
Example 17
/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);

	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(
		CheckpointingOptions.LOCAL_RECOVERY.key(),
		CheckpointingOptions.LOCAL_RECOVERY.defaultValue());

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			configuredMemory = MemorySize.parse(configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		configuredMemory = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(
		configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}