org.apache.flink.configuration.MemorySize Java Examples

The following examples show how to use org.apache.flink.configuration.MemorySize. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskExecutorProcessUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static TaskExecutorProcessSpec processSpecFromWorkerResourceSpec(
	final Configuration config, final WorkerResourceSpec workerResourceSpec) {

	final MemorySize frameworkHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkOffHeapMemorySize(config);

	final TaskExecutorFlinkMemory flinkMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		workerResourceSpec.getTaskHeapSize(),
		workerResourceSpec.getTaskOffHeapSize(),
		workerResourceSpec.getNetworkMemSize(),
		workerResourceSpec.getManagedMemSize());

	final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead =
		PROCESS_MEMORY_UTILS.deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(
			config, flinkMemory.getTotalFlinkMemorySize());

	return new TaskExecutorProcessSpec(workerResourceSpec.getCpuCores(), flinkMemory, jvmMetaspaceAndOverhead);
}
 
Example #2
Source File: WebFrontendITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}

	// !!DO NOT REMOVE!! next line is required for tests
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));

	return config;
}
 
Example #3
Source File: ShuffleEnvironmentContext.java    From flink with Apache License 2.0 6 votes vote down vote up
public ShuffleEnvironmentContext(
		Configuration configuration,
		ResourceID taskExecutorResourceId,
		MemorySize networkMemorySize,
		boolean localCommunicationOnly,
		InetAddress hostAddress,
		TaskEventPublisher eventPublisher,
		MetricGroup parentMetricGroup,
		Executor ioExecutor) {
	this.configuration = checkNotNull(configuration);
	this.taskExecutorResourceId = checkNotNull(taskExecutorResourceId);
	this.networkMemorySize = networkMemorySize;
	this.localCommunicationOnly = localCommunicationOnly;
	this.hostAddress = checkNotNull(hostAddress);
	this.eventPublisher = checkNotNull(eventPublisher);
	this.parentMetricGroup = checkNotNull(parentMetricGroup);
	this.ioExecutor = ioExecutor;
}
 
Example #4
Source File: ResourceProfile.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new ResourceProfile.
 *
 * @param cpuCores The number of CPU cores (possibly fractional, i.e., 0.2 cores)
 * @param taskHeapMemory The size of the task heap memory.
 * @param taskOffHeapMemory The size of the task off-heap memory.
 * @param managedMemory The size of the managed memory.
 * @param networkMemory The size of the network memory.
 * @param extendedResources The extended resources such as GPU and FPGA
 */
private ResourceProfile(
		final Resource cpuCores,
		final MemorySize taskHeapMemory,
		final MemorySize taskOffHeapMemory,
		final MemorySize managedMemory,
		final MemorySize networkMemory,
		final Map<String, Resource> extendedResources) {

	checkNotNull(cpuCores);
	checkArgument(cpuCores instanceof CPUResource, "cpuCores must be CPUResource");

	this.cpuCores = cpuCores;
	this.taskHeapMemory = checkNotNull(taskHeapMemory);
	this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
	this.managedMemory = checkNotNull(managedMemory);
	this.networkMemory = checkNotNull(networkMemory);
	if (extendedResources != null) {
		this.extendedResources.putAll(extendedResources);
	}
}
 
Example #5
Source File: SavepointMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 
Example #6
Source File: HAQueryableStateFsBackendITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example #7
Source File: ClassLoaderITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example #8
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadDynamicalProperties() throws IOException, FlinkParseException {
	final File tmpDir = temporaryFolder.newFolder();
	final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
	final PrintWriter pw1 = new PrintWriter(confFile);
	final long managedMemory = 1024 * 1024 * 256;
	pw1.println(JobManagerOptions.ADDRESS.key() + ": localhost");
	pw1.println(TaskManagerOptions.MANAGED_MEMORY_SIZE.key() + ": " + managedMemory + "b");
	pw1.close();

	final String jmHost = "host1";
	final int jmPort = 12345;
	String[] args = new String[] {
		"--configDir", tmpDir.toString(),
		"-D" + JobManagerOptions.ADDRESS.key() + "=" + jmHost,
		"-D" + JobManagerOptions.PORT.key() + "=" + jmPort
	};
	Configuration configuration = TaskManagerRunner.loadConfiguration(args);
	assertEquals(MemorySize.parse(managedMemory + "b"), configuration.get(TaskManagerOptions.MANAGED_MEMORY_SIZE));
	assertEquals(jmHost, configuration.get(JobManagerOptions.ADDRESS));
	assertEquals(jmPort, configuration.getInteger(JobManagerOptions.PORT));
}
 
Example #9
Source File: JobManagerProcessUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOffHeapMemoryDerivedFromJvmHeapAndTotalFlinkMemory() {
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);
	MemorySize defaultOffHeap = JobManagerOptions.OFF_HEAP_MEMORY.defaultValue();
	MemorySize expectedOffHeap = MemorySize.ofMebiBytes(100).add(defaultOffHeap);
	MemorySize totalFlinkMemory = jvmHeap.add(expectedOffHeap);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmDirectMemorySize(), is(expectedOffHeap));
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The Off-Heap Memory size (%s) is derived the configured Total Flink Memory size (%s) minus " +
				"the configured JVM Heap Memory size (%s). The default Off-Heap Memory size (%s) is ignored.",
			expectedOffHeap.toHumanReadableString(),
			totalFlinkMemory.toHumanReadableString(),
			jvmHeap.toHumanReadableString(),
			defaultOffHeap.toHumanReadableString()))));
}
 
Example #10
Source File: NettyShuffleEnvironmentConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 
Example #11
Source File: NettyShuffleEnvironmentConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 
Example #12
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #13
Source File: TaskManagerServicesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 
Example #14
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigManagedMemoryLegacySize() {
	final MemorySize managedMemSize = MemorySize.parse("100m");

	@SuppressWarnings("deprecation")
	final ConfigOption<MemorySize> legacyOption = TaskManagerOptions.MANAGED_MEMORY_SIZE;

	Configuration conf = new Configuration();
	conf.set(legacyOption, managedMemSize);

	// validate in configurations without explicit managed memory size,
	// to avoid checking against overwritten managed memory size
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(managedMemSize)));
}
 
Example #15
Source File: LocalExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 
Example #16
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalConfigurationException.class)
public void testConsistencyCheckOfDerivedNetworkMemoryLessThanMinFails() {
	final Configuration configuration = setupConfigWithFlinkAndTaskHeapToDeriveGivenNetworkMem(500);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("900m"));
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("1000m"));
	// internal validation should fail
	TaskExecutorProcessUtils.processSpecFromConfig(configuration);
}
 
Example #17
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 
Example #18
Source File: ProcessMemoryUtilsTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public static JvmArgTestingProcessMemorySpec generate() {
	return new JvmArgTestingProcessMemorySpec(
		MemorySize.ofMebiBytes(1),
		MemorySize.ofMebiBytes(2),
		MemorySize.ofMebiBytes(3)
	);
}
 
Example #19
Source File: JvmMetaspaceAndOverheadOptions.java    From flink with Apache License 2.0 5 votes vote down vote up
public JvmMetaspaceAndOverheadOptions(
		ConfigOption<MemorySize> jvmMetaspaceOption,
		ConfigOption<MemorySize> jvmOverheadMin,
		ConfigOption<MemorySize> jvmOverheadMax,
		ConfigOption<Float> jvmOverheadFraction) {
	this.jvmMetaspaceOption = jvmMetaspaceOption;
	this.jvmOverheadMin = jvmOverheadMin;
	this.jvmOverheadMax = jvmOverheadMax;
	this.jvmOverheadFraction = jvmOverheadFraction;
}
 
Example #20
Source File: DescriptorProperties.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a Flink {@link MemorySize} under the given key if it exists.
 */
public Optional<MemorySize> getOptionalMemorySize(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES);
		} catch (Exception e) {
			throw new ValidationException("Invalid memory size value for key '" + key + "'.", e);
		}
	});
}
 
Example #21
Source File: DefaultConfigurableOptionsFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public DefaultConfigurableOptionsFactory setWriteBufferSize(String writeBufferSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(writeBufferSize) > 0,
		"Invalid configuration " + writeBufferSize + " for write-buffer size.");

	setInternal(WRITE_BUFFER_SIZE.key(), writeBufferSize);
	return this;
}
 
Example #22
Source File: ResourceProfileTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void includesCPUAndMemoryInToStringIfTheyAreBelowThreshold() {
	ResourceProfile resourceProfile = createResourceProfile(1.0, MemorySize.ofMebiBytes(4));
	assertThat(
		resourceProfile.toString(),
		allOf(
			containsCPUCores(),
			containsTaskHeapMemory()));
}
 
Example #23
Source File: DefaultConfigurableOptionsFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public DefaultConfigurableOptionsFactory setBlockCacheSize(String blockCacheSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(blockCacheSize) > 0,
		"Invalid configuration " + blockCacheSize + " for block cache size.");
	setInternal(BLOCK_CACHE_SIZE.key(), blockCacheSize);

	return this;
}
 
Example #24
Source File: JobManagerProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testJvmHeapExceedsTotalFlinkMemoryFailure() {
	MemorySize totalFlinkMemory = MemorySize.ofMebiBytes(100);
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	validateFail(conf);
}
 
Example #25
Source File: StateBackendLoadingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.parse("20")); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example #26
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigFrameworkHeapMemory() {
	final MemorySize frameworkHeapSize = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkHeapSize(), is(frameworkHeapSize)));
}
 
Example #27
Source File: JobManagerProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLogFailureOfJvmHeapSizeMinSizeVerification() {
	MemorySize jvmHeapMemory = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapMemory);

	JobManagerProcessUtils.processSpecFromConfig(conf);
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The configured or derived JVM heap memory size (%s) is less than its recommended minimum value (%s)",
			jvmHeapMemory.toHumanReadableString(),
			JobManagerOptions.MIN_JVM_HEAP_SIZE.toHumanReadableString()))));
}
 
Example #28
Source File: KubernetesSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {
			"-e", KubernetesSessionClusterExecutor.NAME,
			"-D" + JobManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + jobManagerMemory + "m",
			"-D" + TaskManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + taskManagerMemory + "m",
			"-D" + TaskManagerOptions.NUM_TASK_SLOTS.key() + "=" + slotsPerTaskManager
	};

	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	Configuration executorConfig = cli.getEffectiveConfiguration(args);
	ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #29
Source File: DefaultConfigurableOptionsFactory.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private long getTargetFileSizeBase() {
	return MemorySize.parseBytes(getInternal(TARGET_FILE_SIZE_BASE.key()));
}
 
Example #30
Source File: ResourceProfile.java    From flink with Apache License 2.0 4 votes vote down vote up
private static String addMemorySizeString(String resourceStr, String name, MemorySize size) {
	String comma = resourceStr.isEmpty() ? "" : ", ";
	String memorySizeStr = size == null || size.compareTo(MAX_MEMORY_SIZE_TO_LOG) > 0 ?
		"" : comma + name + '=' + size.toHumanReadableString();
	return resourceStr + memorySizeStr;
}