Java Code Examples for org.apache.flink.configuration.Configuration#getDouble()

The following examples show how to use org.apache.flink.configuration.Configuration#getDouble() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TaskExecutorProcessUtils.java From flink with Apache License 2.0

6 votes

public static CPUResource getCpuCoresWithFallback(final Configuration config, double fallback) {
	final double cpuCores;
	if (config.contains(TaskManagerOptions.CPU_CORES)) {
		cpuCores = config.getDouble(TaskManagerOptions.CPU_CORES);
	} else if (fallback > 0.0) {
		cpuCores = fallback;
	} else {
		cpuCores = config.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);
	}

	if (cpuCores <= 0) {
		throw new IllegalConfigurationException(
			String.format(
				"TaskExecutors need to be started with a positive number of CPU cores. Please configure %s accordingly.",
				TaskManagerOptions.CPU_CORES.key()));
	}

	return new CPUResource(cpuCores);
}

Example 2

Source File: BootstrapTools.java From Flink-CEPplus with Apache License 2.0

5 votes

public static ForkJoinExecutorConfiguration fromConfiguration(final Configuration configuration) {
	final double parallelismFactor = configuration.getDouble(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_FACTOR);
	final int minParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MIN);
	final int maxParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MAX);

	return new ForkJoinExecutorConfiguration(parallelismFactor, minParallelism, maxParallelism);
}

Example 3

Source File: BootstrapTools.java From flink with Apache License 2.0

5 votes

public static ForkJoinExecutorConfiguration fromConfiguration(final Configuration configuration) {
	final double parallelismFactor = configuration.getDouble(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_FACTOR);
	final int minParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MIN);
	final int maxParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MAX);

	return new ForkJoinExecutorConfiguration(parallelismFactor, minParallelism, maxParallelism);
}

Example 4

Source File: BootstrapTools.java From flink with Apache License 2.0

5 votes

public static ForkJoinExecutorConfiguration fromConfiguration(final Configuration configuration) {
	final double parallelismFactor = configuration.getDouble(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_FACTOR);
	final int minParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MIN);
	final int maxParallelism = configuration.getInteger(AkkaOptions.FORK_JOIN_EXECUTOR_PARALLELISM_MAX);

	return new ForkJoinExecutorConfiguration(parallelismFactor, minParallelism, maxParallelism);
}

Example 5

Source File: TaskExecutorResourceUtils.java From flink with Apache License 2.0

5 votes

static TaskExecutorResourceSpec resourceSpecFromConfig(Configuration config) {
	try {
		checkTaskExecutorResourceConfigSet(config);
	} catch (IllegalConfigurationException e) {
		throw new IllegalConfigurationException("Failed to create TaskExecutorResourceSpec", e);
	}
	return new TaskExecutorResourceSpec(
		new CPUResource(config.getDouble(TaskManagerOptions.CPU_CORES)),
		config.get(TaskManagerOptions.TASK_HEAP_MEMORY),
		config.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY),
		config.get(TaskManagerOptions.NETWORK_MEMORY_MIN),
		config.get(TaskManagerOptions.MANAGED_MEMORY_SIZE)
	);
}

Example 6

Source File: MesosTaskManagerParameters.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}

Example 7

Source File: MesosTaskManagerParameters.java From flink with Apache License 2.0

4 votes

/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	int disk = flinkConfig.getInteger(MESOS_RM_TASKS_DISK_MB);

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		disk,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}

Example 8

Source File: KubernetesWorkerResourceSpecFactory.java From flink with Apache License 2.0

4 votes

@VisibleForTesting
static CPUResource getDefaultCpus(Configuration configuration) {
	double fallback = configuration.getDouble(KubernetesConfigOptions.TASK_MANAGER_CPU);
	return TaskExecutorProcessUtils.getCpuCoresWithFallback(configuration, fallback);
}

Example 9

Source File: MesosWorkerResourceSpecFactory.java From flink with Apache License 2.0

4 votes

private static CPUResource getDefaultCpus(Configuration configuration) {
	double fallback = configuration.getDouble(MesosTaskManagerParameters.MESOS_RM_TASKS_CPUS);
	return TaskExecutorProcessUtils.getCpuCoresWithFallback(configuration, fallback);
}

Example 10

Source File: TaskExecutorProcessUtils.java From flink with Apache License 2.0

4 votes

public static double getCpuCoresWithFallbackConfigOption(final Configuration config, ConfigOption<Double> fallbackOption) {
	double fallbackValue = config.getDouble(fallbackOption);
	return getCpuCoresWithFallback(config, fallbackValue).getValue().doubleValue();
}