Java Code Examples for org.apache.flink.configuration.ConfigurationUtils

The following examples show how to use org.apache.flink.configuration.ConfigurationUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: JobMasterConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example 2
Source Project: Flink-CEPplus   Source File: TaskManagerRunner.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static Configuration loadConfiguration(String[] args) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName());
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: ClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example 4
Source Project: Flink-CEPplus   Source File: SessionClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example 5
Source Project: flink   Source File: JobMasterConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example 6
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static Configuration loadConfiguration(String[] args) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName());
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example 7
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example 8
Source Project: flink   Source File: SessionClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example 9
Source Project: flink   Source File: SavepointEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) {
	this.jobID = new JobID();
	this.vertexID = new JobVertexID();
	this.attemptID = new ExecutionAttemptID();
	this.ctx = Preconditions.checkNotNull(ctx);
	this.configuration = Preconditions.checkNotNull(configuration);

	Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism);
	this.maxParallelism = maxParallelism;
	this.indexOfSubtask = indexOfSubtask;

	this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID);
	this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState);
	this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration));
	this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024);
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID);
}
 
Example 10
Source Project: flink   Source File: JobMasterConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example 11
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example 12
Source Project: flink   Source File: SessionClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY);
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCapacity,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 
Example 13
Source Project: flink   Source File: ConfigurationParserUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generate configuration from only the config file and dynamic properties.
 * @param args the commandline arguments
 * @param cmdLineSyntax the syntax for this application
 * @return generated configuration
 * @throws FlinkParseException if the configuration cannot be generated
 */
public static Configuration loadCommonConfiguration(String[] args, String cmdLineSyntax) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(cmdLineSyntax);
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 
Example 14
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 5 votes vote down vote up
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) {
	if (cmd.hasOption(container.getOpt())) { // number of containers is required option!
		LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt());
	}

	// TODO: The number of task manager should be deprecated soon
	final int numberTaskManagers;

	if (cmd.hasOption(container.getOpt())) {
		numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt()));
	} else {
		numberTaskManagers = 1;
	}

	// JobManager Memory
	final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();

	// Task Managers memory
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMB)
		.setTaskManagerMemoryMB(taskManagerMemoryMB)
		.setNumberTaskManagers(numberTaskManagers)
		.setSlotsPerTaskManager(slotsPerTaskManager)
		.createClusterSpecification();
}
 
Example 15
Source Project: Flink-CEPplus   Source File: ClusterSpecification.java    License: Apache License 2.0 5 votes vote down vote up
public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 
Example 16
Source Project: flink   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 5 votes vote down vote up
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) {
	if (cmd.hasOption(container.getOpt())) { // number of containers is required option!
		LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt());
	}

	// TODO: The number of task manager should be deprecated soon
	final int numberTaskManagers;

	if (cmd.hasOption(container.getOpt())) {
		numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt()));
	} else {
		numberTaskManagers = 1;
	}

	// JobManager Memory
	final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();

	// Task Managers memory
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMB)
		.setTaskManagerMemoryMB(taskManagerMemoryMB)
		.setNumberTaskManagers(numberTaskManagers)
		.setSlotsPerTaskManager(slotsPerTaskManager)
		.createClusterSpecification();
}
 
Example 17
Source Project: flink   Source File: ActiveResourceManagerFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static Configuration createActiveResourceManagerConfiguration(Configuration originalConfiguration) {
	final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(originalConfiguration).getMebiBytes();
	final long cutoffMB = ContaineredTaskManagerParameters.calculateCutoffMB(originalConfiguration, taskManagerMemoryMB);
	final long processMemoryBytes = (taskManagerMemoryMB - cutoffMB) << 20; // megabytes to bytes
	final long managedMemoryBytes = TaskManagerServices.getManagedMemoryFromProcessMemory(originalConfiguration, processMemoryBytes);

	final Configuration resourceManagerConfig = new Configuration(originalConfiguration);
	resourceManagerConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemoryBytes + "b");

	return resourceManagerConfig;
}
 
Example 18
Source Project: flink   Source File: ClusterSpecification.java    License: Apache License 2.0 5 votes vote down vote up
public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 
Example 19
Source Project: flink   Source File: BashJavaUtilsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetTmResourceParamsConfigs() throws Exception {
	int expectedResultLines = 2;
	String[] commands = {RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString(), String.valueOf(expectedResultLines)};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	ConfigurationUtils.parseJvmArgString(lines.get(0));
	ConfigurationUtils.parseTmResourceDynamicConfigs(lines.get(1));
}
 
Example 20
Source Project: flink   Source File: BashJavaUtilsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetTmResourceParamsConfigsWithDynamicProperties() throws Exception {
	int expectedResultLines = 2;
	double cpuCores = 39.0;
	String[] commands = {
		RUN_BASH_JAVA_UTILS_CMD_SCRIPT,
		BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString(),
		String.valueOf(expectedResultLines),
		"-D" + TaskManagerOptions.CPU_CORES.key() + "=" + cpuCores};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	Map<String, String> configs = ConfigurationUtils.parseTmResourceDynamicConfigs(lines.get(1));
	assertThat(Double.valueOf(configs.get(TaskManagerOptions.CPU_CORES.key())), is(cpuCores));
}
 
Example 21
Source Project: flink   Source File: BashJavaUtilsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetJmResourceParams() throws Exception {
	int expectedResultLines = 1;
	String[] commands = {
		RUN_BASH_JAVA_UTILS_CMD_SCRIPT,
		BashJavaUtils.Command.GET_JM_RESOURCE_PARAMS.toString(),
		String.valueOf(expectedResultLines)};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	ConfigurationUtils.parseJvmArgString(lines.get(0));
}
 
Example 22
Source Project: flink   Source File: BashJavaUtilsITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetJmResourceParamsWithDynamicProperties() throws Exception {
	int expectedResultLines = 1;
	long metaspace = 123456789L;
	String[] commands = {
		RUN_BASH_JAVA_UTILS_CMD_SCRIPT,
		BashJavaUtils.Command.GET_JM_RESOURCE_PARAMS.toString(),
		String.valueOf(expectedResultLines),
		"-D" + JobManagerOptions.JVM_METASPACE.key() + "=" + metaspace + "b"};
	List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator()));

	assertThat(lines.size(), is(expectedResultLines));
	Map<String, String> params = ConfigurationUtils.parseJvmArgString(lines.get(0));
	assertThat(Long.valueOf(params.get("-XX:MaxMetaspaceSize=")), is(metaspace));
}
 
Example 23
protected PythonEnvironmentManager createPythonEnvironmentManager() throws IOException {
	PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(
		config, getRuntimeContext().getDistributedCache());
	PythonEnv pythonEnv = getPythonEnv();
	if (pythonEnv.getExecType() == PythonEnv.ExecType.PROCESS) {
		return new ProcessPythonEnvironmentManager(
			dependencyInfo,
			ConfigurationUtils.splitPaths(System.getProperty("java.io.tmpdir")),
			System.getenv());
	} else {
		throw new UnsupportedOperationException(String.format(
			"Execution type '%s' is not supported.", pythonEnv.getExecType()));
	}
}
 
Example 24
Source Project: flink   Source File: StandaloneResourceManagerFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}
 
Example 25
Source Project: flink   Source File: JobConfigInfo.java    License: Apache License 2.0 5 votes vote down vote up
public static ExecutionConfigInfo from(ArchivedExecutionConfig archivedExecutionConfig) {
	return new ExecutionConfigInfo(
		archivedExecutionConfig.getExecutionMode(),
		archivedExecutionConfig.getRestartStrategyDescription(),
		archivedExecutionConfig.getParallelism(),
		archivedExecutionConfig.getObjectReuseEnabled(),
		ConfigurationUtils.hideSensitiveValues(archivedExecutionConfig.getGlobalJobParameters()));
}
 
Example 26
Source Project: flink   Source File: ClusterConfigurationInfo.java    License: Apache License 2.0 5 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	final ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());
	final Map<String, String> configurationWithHiddenSensitiveValues = ConfigurationUtils.hideSensitiveValues(config.toMap());

	for (Map.Entry<String, String> keyValuePair : configurationWithHiddenSensitiveValues.entrySet()) {
		clusterConfig.add(new ClusterConfigurationInfoEntry(keyValuePair.getKey(), keyValuePair.getValue()));
	}

	return clusterConfig;
}
 
Example 27
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 5 votes vote down vote up
protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService(
				configuration,
				configuration.getString(JobManagerOptions.ADDRESS),
				getRPCPortRange(configuration),
				configuration.getString(JobManagerOptions.BIND_HOST),
				configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				ClusterEntrypointUtils.getPoolSize(configuration),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration, pluginManager);

			final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress());
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			final String hostname = RpcUtils.getHostname(commonRpcService);

			processMetricGroup = MetricUtils.instantiateProcessMetricGroup(
				metricRegistry,
				hostname,
				ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
 
Example 28
Source Project: flink   Source File: TaskExecutorProcessUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGenerateDynamicConfigurations() {
	String dynamicConfigsStr = TaskExecutorProcessUtils.generateDynamicConfigsStr(TM_RESOURCE_SPEC);
	Map<String, String> configs = ConfigurationUtils.parseTmResourceDynamicConfigs(dynamicConfigsStr);

	assertThat(new CPUResource(Double.valueOf(configs.get(TaskManagerOptions.CPU_CORES.key()))), is(TM_RESOURCE_SPEC.getCpuCores()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkOffHeapMemorySize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskOffHeapSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MAX.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MIN.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize()));
	assertThat(MemorySize.parse(configs.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key())), is(TM_RESOURCE_SPEC.getManagedMemorySize()));
}
 
Example 29
Source Project: flink   Source File: BootstrapToolsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetEnvironmentVariables() {
	Configuration testConf = new Configuration();
	testConf.setString("containerized.master.env.LD_LIBRARY_PATH", "/usr/lib/native");

	Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf);

	Assert.assertEquals(1, res.size());
	Map.Entry<String, String> entry = res.entrySet().iterator().next();
	Assert.assertEquals("LD_LIBRARY_PATH", entry.getKey());
	Assert.assertEquals("/usr/lib/native", entry.getValue());
}
 
Example 30
Source Project: flink   Source File: BootstrapToolsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetEnvironmentVariablesErroneous() {
	Configuration testConf = new Configuration();
	testConf.setString("containerized.master.env.", "/usr/lib/native");

	Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf);

	Assert.assertEquals(0, res.size());
}