Java Code Examples for org.apache.flink.configuration.Configuration#addAll()

The following examples show how to use org.apache.flink.configuration.Configuration#addAll() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 
Example 2
private void validateInConfigWithExplicitTaskHeapAndManagedMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit task heap and managed memory size.");
	final Configuration config = configWithExplicitTaskHeapAndManageMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE));
	assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(MANAGED_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 
Example 3
Source Project: Flink-CEPplus   File: FlinkDistribution.java    License: Apache License 2.0 5 votes vote down vote up
public void appendConfiguration(Configuration config) throws IOException {
	final Configuration mergedConfig = new Configuration();
	mergedConfig.addAll(defaultConfig);
	mergedConfig.addAll(config);

	final List<String> configurationLines = mergedConfig.toMap().entrySet().stream()
		.map(entry -> entry.getKey() + ": " + entry.getValue())
		.collect(Collectors.toList());

	Files.write(conf.resolve("flink-conf.yaml"), configurationLines);
}
 
Example 4
private void validateInConfigWithExplicitTotalFlinkAndJvmHeapMem(
		Configuration customConfig,
		Consumer<JobManagerProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink and jvm heap memory size.");
	Configuration config = configWithExplicitTotalFlinkAndJvmHeapMem();
	config.addAll(customConfig);
	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(config);
	assertThat(jobManagerProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(JVM_HEAP_SIZE));
	validateFunc.accept(jobManagerProcessSpec);
}
 
Example 5
private void validateInConfigWithExplicitTotalProcessMem(
	Configuration customConfig, Consumer<JobManagerProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total process memory size.");
	Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(config);
	assertThat(jobManagerProcessSpec.getTotalProcessMemorySize(), is(TOTAL_PROCESS_MEM_SIZE));
	validateFunc.accept(jobManagerProcessSpec);
}
 
Example 6
private void validateInConfigWithExplicitTotalFlinkMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink memory size.");
	final Configuration config = configWithExplicitTotalFlinkMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 
Example 7
Source Project: flink   File: BackPressureITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.addAll(createBackPressureSamplingConfiguration());
	configuration.addAll(createNetworkBufferConfiguration());

	final TestingMiniClusterConfiguration testingMiniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(NUM_TASKS)
		.setConfiguration(configuration)
		.build();

	testingMiniCluster = new TestingMiniCluster(testingMiniClusterConfiguration);
	testingMiniCluster.start();
	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get();
}
 
Example 8
private Configuration combineWithGlobalJobConfiguration(Configuration parameters) {
  Configuration combined = new Configuration();
  combined.addAll(parameters);

  GlobalJobParameters globalJobParameters =
      getRuntimeContext().getExecutionConfig().getGlobalJobParameters();

  Preconditions.checkState(globalJobParameters instanceof Configuration);
  Configuration configuration = (Configuration) globalJobParameters;

  combined.addAll(configuration);
  return combined;
}
 
Example 9
private void validateInConfigWithExplicitTotalFlinkAndTaskHeapMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink and task heap memory size.");
	final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	assertThat(taskExecutorProcessSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 
Example 10
Source Project: flink   File: FlinkDistribution.java    License: Apache License 2.0 5 votes vote down vote up
public void appendConfiguration(Configuration config) throws IOException {
	final Configuration mergedConfig = new Configuration();
	mergedConfig.addAll(defaultConfig);
	mergedConfig.addAll(config);

	final List<String> configurationLines = mergedConfig.toMap().entrySet().stream()
		.map(entry -> entry.getKey() + ": " + entry.getValue())
		.collect(Collectors.toList());

	Files.write(conf.resolve("flink-conf.yaml"), configurationLines);
}
 
Example 11
Source Project: flink   File: JobRetrievalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 
Example 12
/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);

    JobGraph jobGraph = streamGraph.getJobGraph();

    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());

    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS,
            jobGraph.getMaximumParallelism());

    // add (and override) the settings with what the user defined
    configuration.addAll(_conf);

    _exec = new LocalFlinkMiniCluster(configuration, true);

    try {
        _exec.start();
        return _exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled());
    } finally {
        transformations.clear();
        _exec.stop();
        _exec = null;
    }
}
 
Example 13
private void validateInConfigWithExplicitTotalProcessMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total process memory size.");
	final Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalProcessMemorySize(), is(TOTAL_PROCESS_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 
Example 14
private void validateFailInConfigWithExplicitTotalFlinkAndTaskHeapMem(final Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink and task heap memory size.");
	final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem();
	config.addAll(customConfig);
	validateFail(config);
}
 
Example 15
private void validateFailInConfigWithExplicitTotalFlinkMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink memory size.");
	Configuration config = configWithExplicitTotalFlinkMem();
	config.addAll(customConfig);
	validateFail(config);
}
 
Example 16
private void validateFailInConfigWithExplicitTotalFlinkAndJvmHeapMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink and jvm heap memory size.");
	Configuration config = configWithExplicitTotalFlinkAndJvmHeapMem();
	config.addAll(customConfig);
	validateFail(config);
}
 
Example 17
private void validateFailInConfigWithExplicitTotalProcessMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total process memory size.");
	Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	validateFail(config);
}
 
Example 18
private void validateFailInConfigWithExplicitJvmHeap(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit jvm heap.");
	Configuration config = configWithExplicitJvmHeap();
	config.addAll(customConfig);
	validateFail(config);
}
 
Example 19
Source Project: flink   File: LocalStreamEnvironment.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Executes the JobGraph of the on a mini cluster of ClusterUtil with a user
 * specified name.
 *
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}
 
Example 20
private void validateFailInConfigWithExplicitTotalProcessMem(final Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total process memory size.");
	final Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	validateFail(config);
}