Java Code Examples for org.apache.flink.configuration.Configuration#set()

The following examples show how to use org.apache.flink.configuration.Configuration#set() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 
Example 2
Source Project: flink   File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 3
Source Project: flink   File: PythonDependencyInfoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testParsePythonFiles() {
	// Skip this test on Windows as we can not control the Window Driver letters.
	Assume.assumeFalse(OperatingSystem.isWindows());

	Configuration config = new Configuration();
	Map<String, String> pythonFiles = new HashMap<>();
	pythonFiles.put("python_file_{SHA256_0}", "test_file1.py");
	pythonFiles.put("python_file_{SHA256_1}", "test_file2.py");
	config.set(PythonDependencyUtils.PYTHON_FILES, pythonFiles);
	PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);

	Map<String, String> expected = new HashMap<>();
	expected.put("/distributed_cache/file0", "test_file1.py");
	expected.put("/distributed_cache/file1", "test_file2.py");
	assertEquals(expected, dependencyInfo.getPythonFiles());
}
 
Example 4
@Test
public void testConfigurationWithDeprecatedOptions() {
	final Configuration configuration = new Configuration();

	final Duration refusedRegistrationBackoff = Duration.ofMinutes(42L);
	final Duration registrationMaxBackoff = Duration.ofSeconds(1L);
	final Duration initialRegistrationBackoff = Duration.ofHours(1337L);

	configuration.set(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF, refusedRegistrationBackoff);
	configuration.set(TaskManagerOptions.REGISTRATION_MAX_BACKOFF, registrationMaxBackoff);
	configuration.set(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF, initialRegistrationBackoff);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(ClusterOptions.REFUSED_REGISTRATION_DELAY.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(ClusterOptions.MAX_REGISTRATION_TIMEOUT.defaultValue()));
}
 
Example 5
Source Project: flink   File: TaskExecutorResourceUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void adjustNetworkMemoryForLocalExecution(Configuration config) {
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MIN, config.get(TaskManagerOptions.NETWORK_MEMORY_MAX));
	}
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MAX, config.get(TaskManagerOptions.NETWORK_MEMORY_MIN));
	}
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MIN, DEFAULT_SHUFFLE_MEMORY_SIZE);
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MAX, DEFAULT_SHUFFLE_MEMORY_SIZE);
}
 
Example 6
@Test
public void testConfigBothNewOptionAndLegacyHeapSize() {
	MemorySize newOptionValue = MemorySize.parse("1g");
	MemorySize legacyHeapSize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(getNewOptionForLegacyHeapOption(), newOptionValue);
	conf.set(legacyMemoryOptions.getHeap(), legacyHeapSize);

	testConfigLegacyHeapMemory(conf, newOptionValue);
}
 
Example 7
@Test
public void testConfigJvmHeapMemory() {
	MemorySize jvmHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapSize);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(jvmHeapSize));
}
 
Example 8
Source Project: flink   File: PythonConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonFrameworkMemorySize() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_FRAMEWORK_MEMORY_SIZE, "100mb");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonFrameworkMemorySize(), is(equalTo("100mb")));
}
 
Example 9
@Test
public void testConfigFrameworkOffHeapMemory() {
	final MemorySize frameworkOffHeapSize = MemorySize.parse("10m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, frameworkOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkOffHeapMemorySize(), is(frameworkOffHeapSize)));
}
 
Example 10
@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 
Example 11
@Test
public void testConfigTotalProcessMemorySize() {
	MemorySize totalProcessMemorySize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemorySize);

	T processSpec = processSpecFromConfig(conf);
	assertThat(processSpec.getTotalProcessMemorySize(), is(totalProcessMemorySize));
}
 
Example 12
@Test
public void testConfigNetworkMemoryRangeFailure() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);

	validateFailInAllConfigurations(conf);
}
 
Example 13
/**
 * Tests that the TaskManagerRunner startup fails synchronously when the memory configuration is wrong.
 */
@Test(expected = IllegalConfigurationException.class)
public void testMemoryConfigWrong() throws Exception {
	Configuration cfg = createFlinkConfiguration();

	// something invalid
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("100m"));
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("10m"));
	startTaskManager(
		cfg,
		rpcService,
		highAvailabilityServices);
}
 
Example 14
@Test
public void testOnlyOneJobAllowedWithStaticJobIdAndHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 
Example 15
@Override
protected void configWithFineGrainedOptions(Configuration configuration, MemorySize totalFlinkMemorySize) {
	MemorySize heapSize = new MemorySize(totalFlinkMemorySize.getBytes() / 2);
	MemorySize offHeapSize = totalFlinkMemorySize.subtract(heapSize);
	configuration.set(JobManagerOptions.JVM_HEAP_MEMORY, heapSize);
	configuration.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapSize);
}
 
Example 16
Source Project: flink   File: LocalExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 
Example 17
Source Project: flink   File: PythonConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonExec() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_EXECUTABLE, "/usr/local/bin/python3");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonExec(), is(equalTo("/usr/local/bin/python3")));
}
 
Example 18
Source Project: flink   File: CancelingTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}
 
Example 19
Source Project: flink   File: SecurityUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify that we pick the first valid security context.
 */
@Test
public void testSecurityContextShouldPickFirstIfBothCompatible() throws Exception {
	Configuration testFlinkConf = new Configuration();

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName(),
					TestSecurityContextFactory.class.getCanonicalName()));

	SecurityConfiguration testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			AnotherCompatibleTestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					TestSecurityContextFactory.class.getCanonicalName(),
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName()));

	testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			TestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());
}
 
Example 20
Source Project: flink   File: AccumulatorErrorITCase.java    License: Apache License 2.0 4 votes vote down vote up
public static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));
	return config;
}