Java Code Examples for org.apache.flink.configuration.MemorySize

The following examples show how to use org.apache.flink.configuration.MemorySize. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: ResourceProfile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new ResourceProfile.
 *
 * @param cpuCores The number of CPU cores (possibly fractional, i.e., 0.2 cores)
 * @param taskHeapMemory The size of the task heap memory.
 * @param taskOffHeapMemory The size of the task off-heap memory.
 * @param managedMemory The size of the managed memory.
 * @param networkMemory The size of the network memory.
 * @param extendedResources The extended resources such as GPU and FPGA
 */
private ResourceProfile(
		final Resource cpuCores,
		final MemorySize taskHeapMemory,
		final MemorySize taskOffHeapMemory,
		final MemorySize managedMemory,
		final MemorySize networkMemory,
		final Map<String, Resource> extendedResources) {

	checkNotNull(cpuCores);
	checkArgument(cpuCores instanceof CPUResource, "cpuCores must be CPUResource");

	this.cpuCores = cpuCores;
	this.taskHeapMemory = checkNotNull(taskHeapMemory);
	this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
	this.managedMemory = checkNotNull(managedMemory);
	this.networkMemory = checkNotNull(networkMemory);
	if (extendedResources != null) {
		this.extendedResources.putAll(extendedResources);
	}
}
 
Example 2
Source Project: flink   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 3
Source Project: flink   Source File: JobManagerProcessUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOffHeapMemoryDerivedFromJvmHeapAndTotalFlinkMemory() {
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);
	MemorySize defaultOffHeap = JobManagerOptions.OFF_HEAP_MEMORY.defaultValue();
	MemorySize expectedOffHeap = MemorySize.ofMebiBytes(100).add(defaultOffHeap);
	MemorySize totalFlinkMemory = jvmHeap.add(expectedOffHeap);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmDirectMemorySize(), is(expectedOffHeap));
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The Off-Heap Memory size (%s) is derived the configured Total Flink Memory size (%s) minus " +
				"the configured JVM Heap Memory size (%s). The default Off-Heap Memory size (%s) is ignored.",
			expectedOffHeap.toHumanReadableString(),
			totalFlinkMemory.toHumanReadableString(),
			jvmHeap.toHumanReadableString(),
			defaultOffHeap.toHumanReadableString()))));
}
 
Example 4
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 5
Source Project: Flink-CEPplus   Source File: TaskManagerServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 
Example 6
Source Project: flink   Source File: SavepointMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 
Example 7
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 
Example 8
/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 
Example 9
Source Project: flink   Source File: TaskManagerRunnerConfigurationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLoadDynamicalProperties() throws IOException, FlinkParseException {
	final File tmpDir = temporaryFolder.newFolder();
	final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
	final PrintWriter pw1 = new PrintWriter(confFile);
	final long managedMemory = 1024 * 1024 * 256;
	pw1.println(JobManagerOptions.ADDRESS.key() + ": localhost");
	pw1.println(TaskManagerOptions.MANAGED_MEMORY_SIZE.key() + ": " + managedMemory + "b");
	pw1.close();

	final String jmHost = "host1";
	final int jmPort = 12345;
	String[] args = new String[] {
		"--configDir", tmpDir.toString(),
		"-D" + JobManagerOptions.ADDRESS.key() + "=" + jmHost,
		"-D" + JobManagerOptions.PORT.key() + "=" + jmPort
	};
	Configuration configuration = TaskManagerRunner.loadConfiguration(args);
	assertEquals(MemorySize.parse(managedMemory + "b"), configuration.get(TaskManagerOptions.MANAGED_MEMORY_SIZE));
	assertEquals(jmHost, configuration.get(JobManagerOptions.ADDRESS));
	assertEquals(jmPort, configuration.getInteger(JobManagerOptions.PORT));
}
 
Example 10
Source Project: flink   Source File: HAQueryableStateFsBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example 11
Source Project: flink   Source File: ShuffleEnvironmentContext.java    License: Apache License 2.0 6 votes vote down vote up
public ShuffleEnvironmentContext(
		Configuration configuration,
		ResourceID taskExecutorResourceId,
		MemorySize networkMemorySize,
		boolean localCommunicationOnly,
		InetAddress hostAddress,
		TaskEventPublisher eventPublisher,
		MetricGroup parentMetricGroup,
		Executor ioExecutor) {
	this.configuration = checkNotNull(configuration);
	this.taskExecutorResourceId = checkNotNull(taskExecutorResourceId);
	this.networkMemorySize = networkMemorySize;
	this.localCommunicationOnly = localCommunicationOnly;
	this.hostAddress = checkNotNull(hostAddress);
	this.eventPublisher = checkNotNull(eventPublisher);
	this.parentMetricGroup = checkNotNull(parentMetricGroup);
	this.ioExecutor = ioExecutor;
}
 
Example 12
Source Project: flink   Source File: WebFrontendITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}

	// !!DO NOT REMOVE!! next line is required for tests
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));

	return config;
}
 
Example 13
Source Project: flink   Source File: TaskExecutorProcessUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static TaskExecutorProcessSpec processSpecFromWorkerResourceSpec(
	final Configuration config, final WorkerResourceSpec workerResourceSpec) {

	final MemorySize frameworkHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkOffHeapMemorySize(config);

	final TaskExecutorFlinkMemory flinkMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		workerResourceSpec.getTaskHeapSize(),
		workerResourceSpec.getTaskOffHeapSize(),
		workerResourceSpec.getNetworkMemSize(),
		workerResourceSpec.getManagedMemSize());

	final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead =
		PROCESS_MEMORY_UTILS.deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(
			config, flinkMemory.getTotalFlinkMemorySize());

	return new TaskExecutorProcessSpec(workerResourceSpec.getCpuCores(), flinkMemory, jvmMetaspaceAndOverhead);
}
 
Example 14
Source Project: flink   Source File: KubernetesSessionCliTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {
			"-e", KubernetesSessionClusterExecutor.NAME,
			"-D" + JobManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + jobManagerMemory + "m",
			"-D" + TaskManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + taskManagerMemory + "m",
			"-D" + TaskManagerOptions.NUM_TASK_SLOTS.key() + "=" + slotsPerTaskManager
	};

	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	Configuration executorConfig = cli.getEffectiveConfiguration(args);
	ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 15
Source Project: Flink-CEPplus   Source File: DescriptorProperties.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns a Flink {@link MemorySize} under the given key if it exists.
 */
public Optional<MemorySize> getOptionalMemorySize(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES);
		} catch (Exception e) {
			throw new ValidationException("Invalid memory size value for key '" + key + "'.", e);
		}
	});
}
 
Example 16
Source Project: flink   Source File: ProcessMemoryUtilsTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static JvmArgTestingProcessMemorySpec generate() {
	return new JvmArgTestingProcessMemorySpec(
		MemorySize.ofMebiBytes(1),
		MemorySize.ofMebiBytes(2),
		MemorySize.ofMebiBytes(3)
	);
}
 
Example 17
Source Project: flink   Source File: JvmMetaspaceAndOverheadOptions.java    License: Apache License 2.0 5 votes vote down vote up
public JvmMetaspaceAndOverheadOptions(
		ConfigOption<MemorySize> jvmMetaspaceOption,
		ConfigOption<MemorySize> jvmOverheadMin,
		ConfigOption<MemorySize> jvmOverheadMax,
		ConfigOption<Float> jvmOverheadFraction) {
	this.jvmMetaspaceOption = jvmMetaspaceOption;
	this.jvmOverheadMin = jvmOverheadMin;
	this.jvmOverheadMax = jvmOverheadMax;
	this.jvmOverheadFraction = jvmOverheadFraction;
}
 
Example 18
Source Project: flink   Source File: TaskExecutorProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 
Example 19
Source Project: flink   Source File: TaskExecutorProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalConfigurationException.class)
public void testConsistencyCheckOfDerivedNetworkMemoryLessThanMinFails() {
	final Configuration configuration = setupConfigWithFlinkAndTaskHeapToDeriveGivenNetworkMem(500);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("900m"));
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("1000m"));
	// internal validation should fail
	TaskExecutorProcessUtils.processSpecFromConfig(configuration);
}
 
Example 20
Source Project: flink   Source File: TaskExecutorProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigManagedMemoryLegacySize() {
	final MemorySize managedMemSize = MemorySize.parse("100m");

	@SuppressWarnings("deprecation")
	final ConfigOption<MemorySize> legacyOption = TaskManagerOptions.MANAGED_MEMORY_SIZE;

	Configuration conf = new Configuration();
	conf.set(legacyOption, managedMemSize);

	// validate in configurations without explicit managed memory size,
	// to avoid checking against overwritten managed memory size
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(managedMemSize)));
}
 
Example 21
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 5 votes vote down vote up
public DefaultConfigurableOptionsFactory setWriteBufferSize(String writeBufferSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(writeBufferSize) > 0,
		"Invalid configuration " + writeBufferSize + " for write-buffer size.");

	setInternal(WRITE_BUFFER_SIZE.key(), writeBufferSize);
	return this;
}
 
Example 22
Source Project: flink   Source File: ResourceProfileTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void includesCPUAndMemoryInToStringIfTheyAreBelowThreshold() {
	ResourceProfile resourceProfile = createResourceProfile(1.0, MemorySize.ofMebiBytes(4));
	assertThat(
		resourceProfile.toString(),
		allOf(
			containsCPUCores(),
			containsTaskHeapMemory()));
}
 
Example 23
Source Project: flink   Source File: DefaultConfigurableOptionsFactory.java    License: Apache License 2.0 5 votes vote down vote up
public DefaultConfigurableOptionsFactory setBlockCacheSize(String blockCacheSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(blockCacheSize) > 0,
		"Invalid configuration " + blockCacheSize + " for block cache size.");
	setInternal(BLOCK_CACHE_SIZE.key(), blockCacheSize);

	return this;
}
 
Example 24
Source Project: flink   Source File: JobManagerProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJvmHeapExceedsTotalFlinkMemoryFailure() {
	MemorySize totalFlinkMemory = MemorySize.ofMebiBytes(100);
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	validateFail(conf);
}
 
Example 25
Source Project: flink   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.parse("20")); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example 26
Source Project: flink   Source File: TaskExecutorProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigFrameworkHeapMemory() {
	final MemorySize frameworkHeapSize = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkHeapSize(), is(frameworkHeapSize)));
}
 
Example 27
Source Project: flink   Source File: JobManagerProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLogFailureOfJvmHeapSizeMinSizeVerification() {
	MemorySize jvmHeapMemory = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapMemory);

	JobManagerProcessUtils.processSpecFromConfig(conf);
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The configured or derived JVM heap memory size (%s) is less than its recommended minimum value (%s)",
			jvmHeapMemory.toHumanReadableString(),
			JobManagerOptions.MIN_JVM_HEAP_SIZE.toHumanReadableString()))));
}
 
Example 28
Source Project: flink   Source File: LocalExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 
Example 29
@Override
protected SinkFunction<Tuple2<Boolean, Row>> createSinkFunction(
		List<Host> hosts,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		ElasticsearchUpsertSinkFunction upsertSinkFunction) {

	final List<HttpHost> httpHosts = hosts.stream()
		.map((host) -> new HttpHost(host.hostname, host.port, host.protocol))
		.collect(Collectors.toList());

	final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> builder = createBuilder(upsertSinkFunction, httpHosts);

	builder.setFailureHandler(failureHandler);

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_MAX_ACTIONS))
		.ifPresent(v -> builder.setBulkFlushMaxActions(Integer.valueOf(v)));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_MAX_SIZE))
		.ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_INTERVAL))
		.ifPresent(v -> builder.setBulkFlushInterval(Long.valueOf(v)));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_ENABLED))
		.ifPresent(v -> builder.setBulkFlushBackoff(Boolean.valueOf(v)));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_TYPE))
		.ifPresent(v -> builder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.valueOf(v)));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_RETRIES))
		.ifPresent(v -> builder.setBulkFlushBackoffRetries(Integer.valueOf(v)));

	Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_DELAY))
		.ifPresent(v -> builder.setBulkFlushBackoffDelay(Long.valueOf(v)));

	builder.setRestClientFactory(
		new DefaultRestClientFactory(
			Optional.ofNullable(sinkOptions.get(REST_MAX_RETRY_TIMEOUT))
				.map(Integer::valueOf)
				.orElse(null),
			sinkOptions.get(REST_PATH_PREFIX)));

	final ElasticsearchSink<Tuple2<Boolean, Row>> sink = builder.build();

	Optional.ofNullable(sinkOptions.get(DISABLE_FLUSH_ON_CHECKPOINT))
		.ifPresent(v -> {
			if (Boolean.valueOf(v)) {
				sink.disableFlushOnCheckpoint();
			}
		});

	return sink;
}
 
Example 30
Source Project: Flink-CEPplus   Source File: DescriptorProperties.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Adds a Flink {@link MemorySize} under the given key.
 */
public void putMemorySize(String key, MemorySize size) {
	checkNotNull(key);
	checkNotNull(size);
	put(key, size.toString());
}