Java Code Examples for org.apache.flink.configuration.Configuration#set()

The following examples show how to use org.apache.flink.configuration.Configuration#set() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RetryingRegistrationConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigurationWithDeprecatedOptions() {
	final Configuration configuration = new Configuration();

	final Duration refusedRegistrationBackoff = Duration.ofMinutes(42L);
	final Duration registrationMaxBackoff = Duration.ofSeconds(1L);
	final Duration initialRegistrationBackoff = Duration.ofHours(1337L);

	configuration.set(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF, refusedRegistrationBackoff);
	configuration.set(TaskManagerOptions.REGISTRATION_MAX_BACKOFF, registrationMaxBackoff);
	configuration.set(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF, initialRegistrationBackoff);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(ClusterOptions.REFUSED_REGISTRATION_DELAY.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(ClusterOptions.MAX_REGISTRATION_TIMEOUT.defaultValue()));
}
 
Example 2
Source File: PythonDependencyInfoTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testParsePythonFiles() {
	// Skip this test on Windows as we can not control the Window Driver letters.
	Assume.assumeFalse(OperatingSystem.isWindows());

	Configuration config = new Configuration();
	Map<String, String> pythonFiles = new HashMap<>();
	pythonFiles.put("python_file_{SHA256_0}", "test_file1.py");
	pythonFiles.put("python_file_{SHA256_1}", "test_file2.py");
	config.set(PythonDependencyUtils.PYTHON_FILES, pythonFiles);
	PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);

	Map<String, String> expected = new HashMap<>();
	expected.put("/distributed_cache/file0", "test_file1.py");
	expected.put("/distributed_cache/file1", "test_file2.py");
	assertEquals(expected, dependencyInfo.getPythonFiles());
}
 
Example 3
Source File: ClassLoaderITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 4
Source File: SavepointMigrationTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 
Example 5
Source File: ApplicationDispatcherBootstrapTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testOnlyOneJobAllowedWithStaticJobIdAndHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 
Example 6
Source File: SecurityUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that we pick the first valid security context.
 */
@Test
public void testSecurityContextShouldPickFirstIfBothCompatible() throws Exception {
	Configuration testFlinkConf = new Configuration();

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName(),
					TestSecurityContextFactory.class.getCanonicalName()));

	SecurityConfiguration testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			AnotherCompatibleTestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					TestSecurityContextFactory.class.getCanonicalName(),
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName()));

	testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			TestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());
}
 
Example 7
Source File: CancelingTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}
 
Example 8
Source File: PythonConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonExec() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_EXECUTABLE, "/usr/local/bin/python3");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonExec(), is(equalTo("/usr/local/bin/python3")));
}
 
Example 9
Source File: LocalExecutorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 
Example 10
Source File: JobManagerProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected void configWithFineGrainedOptions(Configuration configuration, MemorySize totalFlinkMemorySize) {
	MemorySize heapSize = new MemorySize(totalFlinkMemorySize.getBytes() / 2);
	MemorySize offHeapSize = totalFlinkMemorySize.subtract(heapSize);
	configuration.set(JobManagerOptions.JVM_HEAP_MEMORY, heapSize);
	configuration.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapSize);
}
 
Example 11
Source File: TaskExecutorResourceUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void adjustNetworkMemoryForLocalExecution(Configuration config) {
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MIN, config.get(TaskManagerOptions.NETWORK_MEMORY_MAX));
	}
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MAX, config.get(TaskManagerOptions.NETWORK_MEMORY_MIN));
	}
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MIN, DEFAULT_SHUFFLE_MEMORY_SIZE);
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MAX, DEFAULT_SHUFFLE_MEMORY_SIZE);
}
 
Example 12
Source File: TaskManagerRunnerStartupTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the TaskManagerRunner startup fails synchronously when the memory configuration is wrong.
 */
@Test(expected = IllegalConfigurationException.class)
public void testMemoryConfigWrong() throws Exception {
	Configuration cfg = createFlinkConfiguration();

	// something invalid
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("100m"));
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("10m"));
	startTaskManager(
		cfg,
		rpcService,
		highAvailabilityServices);
}
 
Example 13
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigNetworkMemoryRangeFailure() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);

	validateFailInAllConfigurations(conf);
}
 
Example 14
Source File: ProcessMemoryUtilsTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigTotalProcessMemorySize() {
	MemorySize totalProcessMemorySize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemorySize);

	T processSpec = processSpecFromConfig(conf);
	assertThat(processSpec.getTotalProcessMemorySize(), is(totalProcessMemorySize));
}
 
Example 15
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 
Example 16
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigFrameworkOffHeapMemory() {
	final MemorySize frameworkOffHeapSize = MemorySize.parse("10m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, frameworkOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkOffHeapMemorySize(), is(frameworkOffHeapSize)));
}
 
Example 17
Source File: PythonConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonFrameworkMemorySize() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_FRAMEWORK_MEMORY_SIZE, "100mb");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonFrameworkMemorySize(), is(equalTo("100mb")));
}
 
Example 18
Source File: JobManagerProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigJvmHeapMemory() {
	MemorySize jvmHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapSize);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(jvmHeapSize));
}
 
Example 19
Source File: ProcessMemoryUtilsTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigBothNewOptionAndLegacyHeapSize() {
	MemorySize newOptionValue = MemorySize.parse("1g");
	MemorySize legacyHeapSize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(getNewOptionForLegacyHeapOption(), newOptionValue);
	conf.set(legacyMemoryOptions.getHeap(), legacyHeapSize);

	testConfigLegacyHeapMemory(conf, newOptionValue);
}
 
Example 20
Source File: AccumulatorErrorITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
public static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));
	return config;
}