org.apache.flink.configuration.ConfigurationUtils Java Examples

The following examples show how to use org.apache.flink.configuration.ConfigurationUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskManagerRunner.java    From flink with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static Configuration loadConfiguration(String[] args) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName());
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example #2
Source File: SavepointEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) {
	this.jobID = new JobID();
	this.vertexID = new JobVertexID();
	this.attemptID = new ExecutionAttemptID();
	this.ctx = Preconditions.checkNotNull(ctx);
	this.configuration = Preconditions.checkNotNull(configuration);

	Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism);
	this.maxParallelism = maxParallelism;
	this.indexOfSubtask = indexOfSubtask;

	this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID);
	this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState);
	this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration));
	this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024);
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID);
}
 
Example #3
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #4
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example #5
Source File: SessionClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example #6
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example #7
Source File: SessionClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY);
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCapacity,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example #8
Source File: ConfigurationParserUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Generate configuration from only the config file and dynamic properties.
 * @param args the commandline arguments
 * @param cmdLineSyntax the syntax for this application
 * @return generated configuration
 * @throws FlinkParseException if the configuration cannot be generated
 */
public static Configuration loadCommonConfiguration(String[] args, String cmdLineSyntax) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(cmdLineSyntax);
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example #9
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #10
Source File: SessionClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example #11
Source File: ClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example #12
Source File: TaskManagerRunner.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static Configuration loadConfiguration(String[] args) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName());
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example #13
Source File: JobMasterConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #14
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetEnvironmentVariables() {
	Configuration testConf = new Configuration();
	testConf.setString("containerized.master.env.LD_LIBRARY_PATH", "/usr/lib/native");

	Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf);

	Assert.assertEquals(1, res.size());
	Map.Entry<String, String> entry = res.entrySet().iterator().next();
	Assert.assertEquals("LD_LIBRARY_PATH", entry.getKey());
	Assert.assertEquals("/usr/lib/native", entry.getValue());
}
 
Example #15
Source File: TaskExecutorProcessUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateDynamicConfigurations() {
	String dynamicConfigsStr = TaskExecutorProcessUtils.generateDynamicConfigsStr(TM_RESOURCE_SPEC);
	Map<String, String> configs = ConfigurationUtils.parseTmResourceDynamicConfigs(dynamicConfigsStr);

	assertThat(new CPUResource(Double.valueOf(configs.get(TaskManagerOptions.CPU_CORES.key()))), is(TM_RESOURCE_SPEC.getCpuCores()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkOffHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskOffHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MAX.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MIN.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key())), is(TM_RESOURCE_SPEC.getManagedMemorySize()));
}
 
Example #16
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetEnvironmentVariablesErroneous() {
	Configuration testConf = new Configuration();
	testConf.setString("containerized.master.env.", "/usr/lib/native");

	Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf);

	Assert.assertEquals(0, res.size());
}
 
Example #17
Source File: BashJavaUtilsITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetJmResourceParamsWithDynamicProperties() throws Exception {
	int expectedResultLines = 1;
	long metaspace = 123456789L;
	String[] commands = {
		RUN_BASH_JAVA_UTILS_CMD_SCRIPT,
		BashJavaUtils.Command.GET_JM_RESOURCE_PARAMS.toString(),
		String.valueOf(expectedResultLines),
		"-D" + JobManagerOptions.JVM_METASPACE.key() + "=" + metaspace + "b"};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	Map<String, String> params = ConfigurationUtils.parseJvmArgString(lines.get(0));
	assertThat(Long.valueOf(params.get("-XX:MaxMetaspaceSize=")), is(metaspace));
}
 
Example #18
Source File: BashJavaUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testJmLegacyHeapOptionSetsNewJvmHeap() {
	Configuration configuration = new Configuration();
	MemorySize heapSize = MemorySize.ofMebiBytes(10);
	configuration.set(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, heapSize);
	String jvmArgsLine = BashJavaUtils.getJmResourceParams(configuration).get(0);
	Map<String, String> jvmArgs = ConfigurationUtils.parseJvmArgString(jvmArgsLine);
	String heapSizeStr = Long.toString(heapSize.getBytes());
	assertThat(jvmArgs.get("-Xmx"), is(heapSizeStr));
	assertThat(jvmArgs.get("-Xms"), is(heapSizeStr));
}
 
Example #19
Source File: ProcessMemoryUtilsTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateJvmParameters() {
	ProcessMemorySpec spec = JvmArgTestingProcessMemorySpec.generate();
	String jvmParamsStr = ProcessMemoryUtils.generateJvmParametersStr(spec, true);
	Map<String, String> configs = ConfigurationUtils.parseJvmArgString(jvmParamsStr);

	assertThat(configs.size(), is(4));
	assertThat(MemorySize.parse(configs.get("-Xmx")), is(spec.getJvmHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get("-Xms")), is(spec.getJvmHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get("-XX:MaxMetaspaceSize=")), is(spec.getJvmMetaspaceSize()));
	assertThat(MemorySize.parse(configs.get("-XX:MaxDirectMemorySize=")), is(spec.getJvmDirectMemorySize()));
}
 
Example #20
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService(
				configuration,
				configuration.getString(JobManagerOptions.ADDRESS),
				getRPCPortRange(configuration),
				configuration.getString(JobManagerOptions.BIND_HOST),
				configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				ClusterEntrypointUtils.getPoolSize(configuration),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration, pluginManager);

			final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress());
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			final String hostname = RpcUtils.getHostname(commonRpcService);

			processMetricGroup = MetricUtils.instantiateProcessMetricGroup(
				metricRegistry,
				hostname,
				ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
 
Example #21
Source File: ClusterConfigurationInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	final ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());
	final Map<String, String> configurationWithHiddenSensitiveValues = ConfigurationUtils.hideSensitiveValues(config.toMap());

	for (Map.Entry<String, String> keyValuePair : configurationWithHiddenSensitiveValues.entrySet()) {
		clusterConfig.add(new ClusterConfigurationInfoEntry(keyValuePair.getKey(), keyValuePair.getValue()));
	}

	return clusterConfig;
}
 
Example #22
Source File: JobConfigInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ExecutionConfigInfo from(ArchivedExecutionConfig archivedExecutionConfig) {
	return new ExecutionConfigInfo(
		archivedExecutionConfig.getExecutionMode(),
		archivedExecutionConfig.getRestartStrategyDescription(),
		archivedExecutionConfig.getParallelism(),
		archivedExecutionConfig.getObjectReuseEnabled(),
		ConfigurationUtils.hideSensitiveValues(archivedExecutionConfig.getGlobalJobParameters()));
}
 
Example #23
Source File: StandaloneResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}
 
Example #24
Source File: ProcessMemoryUtilsTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateJvmParametersWithoutDirectMemoryLimit() {
	ProcessMemorySpec spec = JvmArgTestingProcessMemorySpec.generate();
	String jvmParamsStr = ProcessMemoryUtils.generateJvmParametersStr(spec, false);
	Map<String, String> configs = ConfigurationUtils.parseJvmArgString(jvmParamsStr);

	assertThat(configs.size(), is(3));
	assertThat(MemorySize.parse(configs.get("-Xmx")), is(spec.getJvmHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get("-Xms")), is(spec.getJvmHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get("-XX:MaxMetaspaceSize=")), is(spec.getJvmMetaspaceSize()));
	assertThat(configs.containsKey("-XX:MaxDirectMemorySize="), is(false));
}
 
Example #25
Source File: ActiveResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public static Configuration createActiveResourceManagerConfiguration(Configuration originalConfiguration) {
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(originalConfiguration).getMebiBytes();
	final long cutoffMB = ContaineredTaskManagerParameters.calculateCutoffMB(originalConfiguration, taskManagerMemoryMB);
	final long processMemoryBytes = (taskManagerMemoryMB - cutoffMB) << 20; // megabytes to bytes
	final long managedMemoryBytes = TaskManagerServices.getManagedMemoryFromProcessMemory(originalConfiguration, processMemoryBytes);

	final Configuration resourceManagerConfig = new Configuration(originalConfiguration);
	resourceManagerConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemoryBytes + "b");

	return resourceManagerConfig;
}
 
Example #26
Source File: FlinkYarnSessionCli.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) {
	if (cmd.hasOption(container.getOpt())) { // number of containers is required option!
		LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt());
	}

	// TODO: The number of task manager should be deprecated soon
	final int numberTaskManagers;

	if (cmd.hasOption(container.getOpt())) {
		numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt()));
	} else {
		numberTaskManagers = 1;
	}

	// JobManager Memory
	final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();

	// Task Managers memory
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMB)
		.setTaskManagerMemoryMB(taskManagerMemoryMB)
		.setNumberTaskManagers(numberTaskManagers)
		.setSlotsPerTaskManager(slotsPerTaskManager)
		.createClusterSpecification();
}
 
Example #27
Source File: ClusterSpecification.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 
Example #28
Source File: FlinkYarnSessionCli.java    From flink with Apache License 2.0 5 votes vote down vote up
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) {
	if (cmd.hasOption(container.getOpt())) { // number of containers is required option!
		LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt());
	}

	// TODO: The number of task manager should be deprecated soon
	final int numberTaskManagers;

	if (cmd.hasOption(container.getOpt())) {
		numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt()));
	} else {
		numberTaskManagers = 1;
	}

	// JobManager Memory
	final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();

	// Task Managers memory
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMB)
		.setTaskManagerMemoryMB(taskManagerMemoryMB)
		.setNumberTaskManagers(numberTaskManagers)
		.setSlotsPerTaskManager(slotsPerTaskManager)
		.createClusterSpecification();
}
 
Example #29
Source File: ClusterSpecification.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 
Example #30
Source File: BashJavaUtilsITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetTmResourceParamsConfigs() throws Exception {
	int expectedResultLines = 2;
	String[] commands = {RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString(), String.valueOf(expectedResultLines)};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	ConfigurationUtils.parseJvmArgString(lines.get(0));
	ConfigurationUtils.parseTmResourceDynamicConfigs(lines.get(1));
}