Java Code Examples for org.apache.flink.configuration.Configuration#getOptional()

The following examples show how to use org.apache.flink.configuration.Configuration#getOptional() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonFiles() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonFiles = configuration.getOptional(PythonOptions.PYTHON_FILES);
	assertThat(defaultPythonFiles, is(equalTo(Optional.empty())));

	final String expectedPythonFiles = "tmp_dir/test1.py,tmp_dir/test2.py";
	configuration.set(PythonOptions.PYTHON_FILES, expectedPythonFiles);

	final String actualPythonFiles = configuration.get(PythonOptions.PYTHON_FILES);
	assertThat(actualPythonFiles, is(equalTo(expectedPythonFiles)));
}
 
Example 2
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonRequirements() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonRequirements = configuration.getOptional(PythonOptions.PYTHON_REQUIREMENTS);
	assertThat(defaultPythonRequirements, is(equalTo(Optional.empty())));

	final String expectedPythonRequirements = "tmp_dir/requirements.txt#tmp_dir/cache";
	configuration.set(PythonOptions.PYTHON_REQUIREMENTS, expectedPythonRequirements);

	final String actualPythonRequirements = configuration.get(PythonOptions.PYTHON_REQUIREMENTS);
	assertThat(actualPythonRequirements, is(equalTo(expectedPythonRequirements)));
}
 
Example 3
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonArchives() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonArchives = configuration.getOptional(PythonOptions.PYTHON_ARCHIVES);
	assertThat(defaultPythonArchives, is(equalTo(Optional.empty())));

	final String expectedPythonArchives = "tmp_dir/py37.zip#venv,tmp_dir/data.zip";
	configuration.set(PythonOptions.PYTHON_ARCHIVES, expectedPythonArchives);

	final String actualPythonArchives = configuration.get(PythonOptions.PYTHON_ARCHIVES);
	assertThat(actualPythonArchives, is(equalTo(expectedPythonArchives)));
}
 
Example 4
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonExecutable() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonExecutable = configuration.getOptional(PythonOptions.PYTHON_EXECUTABLE);
	assertThat(defaultPythonExecutable, is(equalTo(Optional.empty())));

	final String expectedPythonExecutable = "venv/py37/bin/python";
	configuration.set(PythonOptions.PYTHON_EXECUTABLE, expectedPythonExecutable);

	final String actualPythonExecutable = configuration.get(PythonOptions.PYTHON_EXECUTABLE);
	assertThat(actualPythonExecutable, is(equalTo(expectedPythonExecutable)));
}
 
Example 5
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonClientExecutable() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonClientExecutable =
		configuration.getOptional(PythonOptions.PYTHON_CLIENT_EXECUTABLE);
	assertThat(defaultPythonClientExecutable, is(equalTo(Optional.empty())));

	final String expectedPythonClientExecutable = "tmp_dir/test1.py,tmp_dir/test2.py";
	configuration.set(PythonOptions.PYTHON_CLIENT_EXECUTABLE, expectedPythonClientExecutable);

	final String actualPythonClientExecutable = configuration.get(PythonOptions.PYTHON_CLIENT_EXECUTABLE);
	assertThat(actualPythonClientExecutable, is(equalTo(expectedPythonClientExecutable)));
}
 
Example 6
Source Project: flink   File: ExternalResourceUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the map of resource name and amount of all of enabled external resources.
 */
public static Map<String, Long> getExternalResourceAmountMap(Configuration config) {
	final Set<String> resourceSet = getExternalResourceSet(config);
	LOG.info("Enabled external resources: {}", resourceSet);

	if (resourceSet.isEmpty()) {
		return Collections.emptyMap();
	}

	final Map<String, Long> externalResourceAmountMap = new HashMap<>();
	for (String resourceName: resourceSet) {
		final ConfigOption<Long> amountOption =
			key(ExternalResourceOptions.getAmountConfigOptionForResource(resourceName))
				.longType()
				.noDefaultValue();
		final Optional<Long> amountOpt = config.getOptional(amountOption);
		if (!amountOpt.isPresent()) {
			LOG.warn("The amount of the {} should be configured. Will ignore that resource.", resourceName);
		} else if (amountOpt.get() <= 0) {
			LOG.warn("The amount of the {} should be positive while finding {}. Will ignore that resource.", amountOpt.get(), resourceName);
		} else {
			externalResourceAmountMap.put(resourceName, amountOpt.get());
		}
	}

	return externalResourceAmountMap;
}
 
Example 7
Source Project: flink   File: ExternalResourceUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Get the external resources map. The key should be used for deployment specific container request,
 * and values should be the amount of that resource.
 *
 * @param config Configurations
 * @param suffix suffix of config option for deployment specific configuration key
 * @return external resources map, map the amount to the configuration key for deployment specific container request
 */
public static Map<String, Long> getExternalResources(Configuration config, String suffix) {
	final Set<String> resourceSet = getExternalResourceSet(config);
	LOG.info("Enabled external resources: {}", resourceSet);

	if (resourceSet.isEmpty()) {
		return Collections.emptyMap();
	}

	final Map<String, Long> externalResourceConfigs = new HashMap<>();
	for (String resourceName: resourceSet) {
		final ConfigOption<Long> amountOption =
			key(ExternalResourceOptions.getAmountConfigOptionForResource(resourceName))
				.longType()
				.noDefaultValue();
		final ConfigOption<String> configKeyOption =
			key(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(resourceName, suffix))
				.stringType()
				.noDefaultValue();
		final String configKey = config.getString(configKeyOption);
		final Optional<Long> amountOpt = config.getOptional(amountOption);

		if (StringUtils.isNullOrWhitespaceOnly(configKey)) {
			LOG.warn("Could not find valid {} for {}. Will ignore that resource.", configKeyOption.key(), resourceName);
			continue;
		}
		if (!amountOpt.isPresent()) {
			LOG.warn("The amount of the {} should be configured. Will ignore that resource.", resourceName);
			continue;
		} else if (amountOpt.get() <= 0) {
			LOG.warn("The amount of the {} should be positive while finding {}. Will ignore that resource.", amountOpt.get(), resourceName);
			continue;
		}

		if (externalResourceConfigs.put(configKey, amountOpt.get()) != null) {
			LOG.warn("Duplicate config key {} occurred for external resources, the one named {} with amount {} will overwrite the value.", configKey, resourceName, amountOpt);
		} else {
			LOG.info("Add external resource config for {} with key {} value {}.", resourceName, configKey, amountOpt);
		}
	}

	return externalResourceConfigs;
}