Java Code Examples for org.apache.flink.configuration.Configuration#contains()

The following examples show how to use org.apache.flink.configuration.Configuration#contains() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Factory method to startWith a new instance, providing a
 * new instance of {@link MiniCluster}
 *
 * @param parallelism global setting for parallel execution.
 * @return new instance of {@link DataSetTestEnvironment}
 * @throws Exception
 */
public static DataSetTestEnvironment createTestEnvironment(int parallelism) {

    int taskSlots = Runtime.getRuntime().availableProcessors();

    Configuration configuration = new Configuration();
    configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

    if (!configuration.contains(RestOptions.BIND_PORT)) {
        configuration.setString(RestOptions.BIND_PORT, "0");
    }

    int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskSlots);

    MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
            .setConfiguration(configuration)
            .setNumSlotsPerTaskManager(numSlotsPerTaskManager)
            .build();

    MiniCluster miniCluster = new MiniCluster(cfg);

    return new DataSetTestEnvironment(miniCluster, parallelism);
}
 
Example 2
/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataBindPort(Configuration configuration) {
	final int dataBindPort;
	if (configuration.contains(NettyShuffleEnvironmentOptions.DATA_BIND_PORT)) {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_BIND_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_BIND_PORT.key(),
			"Leave config parameter empty to fallback to '" +
				NettyShuffleEnvironmentOptions.DATA_PORT.key() + "' automatically.");
	} else {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
			"Leave config parameter empty or use 0 to let the system choose a port automatically.");
	}
	return dataBindPort;
}
 
Example 3
Source Project: flink   File: ProcessMemoryUtils.java    License: Apache License 2.0 6 votes vote down vote up
private void sanityCheckJvmOverhead(
		Configuration config,
		MemorySize derivedJvmOverheadSize,
		MemorySize totalProcessMemorySize) {
	RangeFraction jvmOverheadRangeFraction = getJvmOverheadRangeFraction(config);
	if (derivedJvmOverheadSize.getBytes() > jvmOverheadRangeFraction.getMaxSize().getBytes() ||
		derivedJvmOverheadSize.getBytes() < jvmOverheadRangeFraction.getMinSize().getBytes()) {
		throw new IllegalConfigurationException("Derived JVM Overhead size ("
			+ derivedJvmOverheadSize.toHumanReadableString() + ") is not in configured JVM Overhead range ["
			+ jvmOverheadRangeFraction.getMinSize().toHumanReadableString() + ", "
			+ jvmOverheadRangeFraction.getMaxSize().toHumanReadableString() + "].");
	}
	if (config.contains(options.getJvmOptions().getJvmOverheadFraction()) &&
		!derivedJvmOverheadSize.equals(totalProcessMemorySize.multiply(jvmOverheadRangeFraction.getFraction()))) {
		LOG.info(
			"The derived JVM Overhead size ({}) does not match " +
				"the configured or default JVM Overhead fraction ({}) from the configured Total Process Memory size ({}). " +
				"The derived JVM OVerhead size will be used.",
			derivedJvmOverheadSize.toHumanReadableString(),
			jvmOverheadRangeFraction.getFraction(),
			totalProcessMemorySize.toHumanReadableString());
	}
}
 
Example 4
@Override
public JobManagerFlinkMemory deriveFromRequiredFineGrainedOptions(Configuration config) {
	MemorySize jvmHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.JVM_HEAP_MEMORY);
	MemorySize offHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.OFF_HEAP_MEMORY);

	if (config.contains(JobManagerOptions.TOTAL_FLINK_MEMORY)) {
		// derive network memory from total flink memory, and check against network min/max
		MemorySize totalFlinkMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.TOTAL_FLINK_MEMORY);
		if (config.contains(JobManagerOptions.OFF_HEAP_MEMORY)) {
			// off-heap memory is explicitly set by user
			sanityCheckTotalFlinkMemory(totalFlinkMemorySize, jvmHeapMemorySize, totalFlinkMemorySize);
		} else {
			// off-heap memory is not explicitly set by user, derive it from Total Flink Memory and JVM Heap
			offHeapMemorySize = deriveOffHeapMemory(jvmHeapMemorySize, totalFlinkMemorySize, offHeapMemorySize);
		}
	}

	return createJobManagerFlinkMemory(jvmHeapMemorySize, offHeapMemorySize);
}
 
Example 5
Source Project: flink   File: LocalExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 
Example 6
/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 
Example 7
Source Project: flink   File: SlotManagerConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
private static Time getSlotRequestTimeout(final Configuration configuration) {
	final long slotRequestTimeoutMs;
	if (configuration.contains(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT)) {
		LOGGER.warn("Config key {} is deprecated; use {} instead.",
			ResourceManagerOptions.SLOT_REQUEST_TIMEOUT,
			JobManagerOptions.SLOT_REQUEST_TIMEOUT);
		slotRequestTimeoutMs = configuration.getLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT);
	} else {
		slotRequestTimeoutMs = configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT);
	}
	return Time.milliseconds(slotRequestTimeoutMs);
}
 
Example 8
@SuppressWarnings("deprecation")
private static void logIfIgnoringOldConfigs(Configuration configuration) {
	if (configuration.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS)) {
		LOG.info("Ignoring old (but still present) network buffer configuration via {}.",
			NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key());
	}
}
 
Example 9
Source Project: Flink-CEPplus   File: BootstrapTools.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set temporary configuration directories if necessary.
 *
 * @param configuration flink config to patch
 * @param defaultDirs in case no tmp directories is set, next directories will be applied
 */
public static void updateTmpDirectoriesInConfiguration(
		Configuration configuration,
		@Nullable String defaultDirs) {
	if (configuration.contains(CoreOptions.TMP_DIRS)) {
		LOG.info("Overriding Fink's temporary file directories with those " +
			"specified in the Flink config: {}", configuration.getValue(CoreOptions.TMP_DIRS));
	} else if (defaultDirs != null) {
		LOG.info("Setting directories for temporary files to: {}", defaultDirs);
		configuration.setString(CoreOptions.TMP_DIRS, defaultDirs);
		configuration.setBoolean(USE_LOCAL_DEFAULT_TMP_DIRS, true);
	}
}
 
Example 10
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
 
Example 11
Source Project: flink   File: ExecutionEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 
Example 12
Source Project: flink   File: TaskExecutorResourceUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> void setConfigOptionToDefaultIfNotSet(
		Configuration config,
		ConfigOption<T> option,
		T defaultValue) {
	if (!config.contains(option)) {
		LOG.info(
			"The configuration option {} required for local execution is not set, setting it to its default value {}",
			option,
			defaultValue);
		config.set(option, defaultValue);
	}
}
 
Example 13
/**
 * Creates a {@link LocalStreamEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(defaultLocalParallelism, conf);
}
 
Example 14
Source Project: flink   File: ExecutionEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 
Example 15
Source Project: flink   File: BootstrapTools.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set temporary configuration directories if necessary.
 *
 * @param configuration flink config to patch
 * @param defaultDirs in case no tmp directories is set, next directories will be applied
 */
public static void updateTmpDirectoriesInConfiguration(
		Configuration configuration,
		@Nullable String defaultDirs) {
	if (configuration.contains(CoreOptions.TMP_DIRS)) {
		LOG.info("Overriding Fink's temporary file directories with those " +
			"specified in the Flink config: {}", configuration.getValue(CoreOptions.TMP_DIRS));
	} else if (defaultDirs != null) {
		LOG.info("Setting directories for temporary files to: {}", defaultDirs);
		configuration.setString(CoreOptions.TMP_DIRS, defaultDirs);
		configuration.setBoolean(USE_LOCAL_DEFAULT_TMP_DIRS, true);
	}
}
 
Example 16
/**
 * Returns whether the new network buffer memory configuration is present in the configuration
 * object, i.e. at least one new parameter is given or the old one is not present.
 *
 * @param config configuration object
 * @return <tt>true</tt> if the new configuration method is used, <tt>false</tt> otherwise
 */
@SuppressWarnings("deprecation")
@VisibleForTesting
public static boolean hasNewNetworkConfig(final Configuration config) {
	return config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION) ||
		config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN) ||
		config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX) ||
		!config.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS);
}
 
Example 17
Source Project: flink   File: MiniClusterResource.java    License: Apache License 2.0 5 votes vote down vote up
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
 
Example 18
private static boolean isTotalFlinkMemorySizeExplicitlyConfigured(final Configuration config) {
	return config.contains(TaskManagerOptions.TOTAL_FLINK_MEMORY);
}
 
Example 19
Source Project: flink   File: YarnEntrypointUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
 
Example 20
private static boolean isTaskHeapMemorySizeExplicitlyConfigured(final Configuration config) {
	return config.contains(TaskManagerOptions.TASK_HEAP_MEMORY);
}