Java Code Examples for org.apache.flink.configuration.Configuration#get()

The following examples show how to use org.apache.flink.configuration.Configuration#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StatefulFunctionsConfig.java From flink-statefun with Apache License 2.0

6 votes

/**
 * Create a new configuration object based on the values set in flink-conf.
 *
 * @param configuration a configuration to read the values from
 */
public StatefulFunctionsConfig(Configuration configuration) {
  StatefulFunctionsConfigValidator.validate(configuration);

  this.factoryType = configuration.get(USER_MESSAGE_SERIALIZER);
  this.flinkJobName = configuration.get(FLINK_JOB_NAME);
  this.feedbackBufferSize = configuration.get(TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING);
  this.maxAsyncOperationsPerTask = configuration.get(ASYNC_MAX_OPERATIONS_PER_TASK);

  for (String key : configuration.keySet()) {
    if (key.startsWith(MODULE_CONFIG_PREFIX)) {
      String value = configuration.get(ConfigOptions.key(key).stringType().noDefaultValue());
      String userKey = key.substring(MODULE_CONFIG_PREFIX.length());
      globalConfigurations.put(userKey, value);
    }
  }
}

Example 2

Source File: StandaloneApplicationClusterConfigurationParserFactoryTest.java From flink with Apache License 2.0

6 votes

@Test
public void testEntrypointClusterConfigWOSavepointSettingsToConfigurationParsing() throws FlinkParseException {
	final JobID jobID = JobID.generate();
	final String[] args = {
			"-c", confDirPath,
			"--job-id", jobID.toHexString()
	};

	final StandaloneApplicationClusterConfiguration clusterConfiguration = commandLineParser.parse(args);
	final Configuration configuration = StandaloneApplicationClusterEntryPoint
			.loadConfigurationFromClusterConfig(clusterConfiguration);

	final String strJobId = configuration.get(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);
	assertThat(JobID.fromHexString(strJobId), is(equalTo(jobID)));
	assertThat(SavepointRestoreSettings.fromConfiguration(configuration), is(equalTo(SavepointRestoreSettings.none())));
}

Example 3

Source File: YARNITCase.java From flink with Apache License 2.0

6 votes

private void checkStagingDirectory(Configuration flinkConfig, ApplicationId appId) throws IOException {
	final List<String> providedLibDirs = flinkConfig.get(YarnConfigOptions.PROVIDED_LIB_DIRS);
	final boolean isProvidedLibDirsConfigured  = providedLibDirs != null && !providedLibDirs.isEmpty();

	try (final FileSystem fs = FileSystem.get(YARN_CONFIGURATION)) {
		final Path stagingDirectory = new Path(fs.getHomeDirectory(), ".flink/" + appId.toString());
		if (isProvidedLibDirsConfigured) {
			assertFalse(
				"The provided lib dirs is set, so the lib directory should not be uploaded to staging directory.",
				fs.exists(new Path(stagingDirectory, flinkLibFolder.getName())));
		} else {
			assertTrue(
				"The lib directory should be uploaded to staging directory.",
				fs.exists(new Path(stagingDirectory, flinkLibFolder.getName())));
		}
	}
}

Example 4

Source File: PythonConfig.java From flink with Apache License 2.0

6 votes

public PythonConfig(Configuration config) {
	maxBundleSize = config.get(PythonOptions.MAX_BUNDLE_SIZE);
	maxBundleTimeMills = config.get(PythonOptions.MAX_BUNDLE_TIME_MILLS);
	maxArrowBatchSize = config.get(PythonOptions.MAX_ARROW_BATCH_SIZE);
	pythonFrameworkMemorySize = config.get(PythonOptions.PYTHON_FRAMEWORK_MEMORY_SIZE);
	pythonDataBufferMemorySize = config.get(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE);
	pythonFilesInfo = config.getOptional(PythonDependencyUtils.PYTHON_FILES).orElse(new HashMap<>());
	pythonRequirementsFileInfo = config.getOptional(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE)
		.orElse(new HashMap<>())
		.get(PythonDependencyUtils.FILE);
	pythonRequirementsCacheDirInfo = config.getOptional(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE)
		.orElse(new HashMap<>())
		.get(PythonDependencyUtils.CACHE);
	pythonArchivesInfo = config.getOptional(PythonDependencyUtils.PYTHON_ARCHIVES).orElse(new HashMap<>());
	pythonExec = config.get(PythonOptions.PYTHON_EXECUTABLE);
	metricEnabled = config.getBoolean(PythonOptions.PYTHON_METRIC_ENABLED);
	isUsingManagedMemory = config.getBoolean(PythonOptions.USE_MANAGED_MEMORY);
}

Example 5

Source File: HiveTableSource.java From flink with Apache License 2.0

5 votes

private DataStream<RowData> createStreamSourceForPartitionTable(
		StreamExecutionEnvironment execEnv,
		TypeInformation<RowData> typeInfo,
		HiveTableInputFormat inputFormat) {
	Configuration configuration = new Configuration();
	catalogTable.getOptions().forEach(configuration::setString);

	String consumeOrderStr = configuration.get(STREAMING_SOURCE_CONSUME_ORDER);
	ConsumeOrder consumeOrder = ConsumeOrder.getConsumeOrder(consumeOrderStr);
	String consumeOffset = configuration.get(STREAMING_SOURCE_CONSUME_START_OFFSET);
	String extractorKind = configuration.get(PARTITION_TIME_EXTRACTOR_KIND);
	String extractorClass = configuration.get(PARTITION_TIME_EXTRACTOR_CLASS);
	String extractorPattern = configuration.get(PARTITION_TIME_EXTRACTOR_TIMESTAMP_PATTERN);
	Duration monitorInterval = configuration.get(STREAMING_SOURCE_MONITOR_INTERVAL);

	HiveContinuousMonitoringFunction monitoringFunction = new HiveContinuousMonitoringFunction(
			hiveShim,
			jobConf,
			tablePath,
			catalogTable,
			execEnv.getParallelism(),
			consumeOrder,
			consumeOffset,
			extractorKind,
			extractorClass,
			extractorPattern,
			monitorInterval.toMillis());

	ContinuousFileReaderOperatorFactory<RowData, TimestampedHiveInputSplit> factory =
			new ContinuousFileReaderOperatorFactory<>(inputFormat);

	String sourceName = "HiveMonitoringFunction";
	SingleOutputStreamOperator<RowData> source = execEnv
			.addSource(monitoringFunction, sourceName)
			.transform("Split Reader: " + sourceName, typeInfo, factory);

	return new DataStreamSource<>(source);
}

Example 6

Source File: TaskExecutorResourceUtils.java From flink with Apache License 2.0

5 votes

static TaskExecutorResourceSpec resourceSpecFromConfig(Configuration config) {
	try {
		checkTaskExecutorResourceConfigSet(config);
	} catch (IllegalConfigurationException e) {
		throw new IllegalConfigurationException("Failed to create TaskExecutorResourceSpec", e);
	}
	return new TaskExecutorResourceSpec(
		new CPUResource(config.getDouble(TaskManagerOptions.CPU_CORES)),
		config.get(TaskManagerOptions.TASK_HEAP_MEMORY),
		config.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY),
		config.get(TaskManagerOptions.NETWORK_MEMORY_MIN),
		config.get(TaskManagerOptions.MANAGED_MEMORY_SIZE)
	);
}

Example 7

Source File: YarnLogConfigUtil.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static Configuration setLogConfigFileInConfig(
		final Configuration configuration,
		final String configurationDirectory) {

	if (configuration.get(YarnConfigOptionsInternal.APPLICATION_LOG_CONFIG_FILE) != null) {
		return configuration;
	}

	discoverLogConfigFile(configurationDirectory).ifPresent(file ->
			configuration.set(YarnConfigOptionsInternal.APPLICATION_LOG_CONFIG_FILE, file.getPath()));
	return configuration;
}

Example 8