Java Code Examples for org.apache.flink.configuration.Configuration#getString()

The following examples show how to use org.apache.flink.configuration.Configuration#getString() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: ClusterConfigurationInfo.java    License: Apache License 2.0 6 votes vote down vote up
public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 
Example 2
Source Project: flink   File: RestHandlerConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public static RestHandlerConfiguration fromConfiguration(Configuration configuration) {
	final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL);

	final int maxCheckpointStatisticCacheEntries = configuration.getInteger(WebOptions.CHECKPOINTS_HISTORY_SIZE);

	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));

	final String rootDir = "flink-web-ui";
	final File webUiDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir);

	final boolean webSubmitEnabled = configuration.getBoolean(WebOptions.SUBMIT_ENABLE);

	return new RestHandlerConfiguration(
		refreshInterval,
		maxCheckpointStatisticCacheEntries,
		timeout,
		webUiDir,
		webSubmitEnabled);
}
 
Example 3
Source Project: flink   File: ReporterSetup.java    License: Apache License 2.0 6 votes vote down vote up
private static Optional<MetricReporter> loadReporter(
		final String reporterName,
		final Configuration reporterConfig,
		final Map<String, MetricReporterFactory> reporterFactories)
		throws ClassNotFoundException, IllegalAccessException, InstantiationException {

	final String reporterClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, null);
	final String factoryClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX, null);

	if (factoryClassName != null) {
		return loadViaFactory(factoryClassName, reporterName, reporterConfig, reporterFactories);
	}

	if (reporterClassName != null) {
		return loadViaReflection(reporterClassName, reporterName, reporterConfig, reporterFactories);
	}

	LOG.warn("No reporter class nor factory set for reporter {}. Metrics might not be exposed/reported.", reporterName);
	return Optional.empty();
}
 
Example 4
public static String stsValidate(String accessId, String accessKey, String localErrorMsg, Configuration properties) {
	String stsRoleArn = properties.getString(BlinkOptions.STS.STS_ROLE_ARN);
	String stsAccessId = properties.getString(BlinkOptions.STS.STS_ACCESS_ID);
	String stsAccessKey = properties.getString(BlinkOptions.STS.STS_ACCESS_KEY);
	String stsUid = properties.getString(BlinkOptions.STS.STS_UID);
	if (BlinkStringUtil.isNotEmpty(accessId, accessKey) || BlinkStringUtil.isNotEmpty(stsRoleArn, stsAccessId, stsAccessKey, stsUid)){
		return null;
	} else if (properties.containsKey(BlinkOptions.STS.STS_ROLE_ARN.key())){
		throw new NotEnoughParamsException(String.format("Lack necessary arguments: {0}", BlinkOptions.STS.STS_PARAMS_HELP_MSG));
	} else {
		throw new NotEnoughParamsException(localErrorMsg);
	}
}
 
Example 5
Source Project: flink   File: HistoryServerArchivist.java    License: Apache License 2.0 5 votes vote down vote up
static HistoryServerArchivist createHistoryServerArchivist(Configuration configuration, JsonArchivist jsonArchivist, Executor ioExecutor) {
	final String configuredArchivePath = configuration.getString(JobManagerOptions.ARCHIVE_DIR);

	if (configuredArchivePath != null) {
		final Path archivePath = WebMonitorUtils.validateAndNormalizeUri(new Path(configuredArchivePath).toUri());

		return new JsonResponseHistoryServerArchivist(jsonArchivist, archivePath, ioExecutor);
	} else {
		return VoidHistoryServerArchivist.INSTANCE;
	}
}
 
Example 6
Source Project: flink   File: ZooKeeperUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link ZooKeeperCheckpointIDCounter} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object
 * @param jobId         ID of job to create the instance for
 * @return {@link ZooKeeperCheckpointIDCounter} instance
 */
public static ZooKeeperCheckpointIDCounter createCheckpointIDCounter(
		CuratorFramework client,
		Configuration configuration,
		JobID jobId) {

	String checkpointIdCounterPath = configuration.getString(
			HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH);

	checkpointIdCounterPath += ZooKeeperJobGraphStore.getPathForJob(jobId);

	return new ZooKeeperCheckpointIDCounter(client, checkpointIdCounterPath, new DefaultLastStateConnectionStateListener());
}
 
Example 7
private static void addStatefulFunctionsConfiguration(Configuration configuration) {
  String parentFirst =
      configuration.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, "");
  if (parentFirst.isEmpty()) {
    parentFirst = Constants.STATEFUL_FUNCTIONS_PACKAGE;
  } else if (parentFirst.endsWith(";")) {
    parentFirst = parentFirst + Constants.STATEFUL_FUNCTIONS_PACKAGE;
  } else {
    parentFirst = parentFirst + ";" + Constants.STATEFUL_FUNCTIONS_PACKAGE;
  }
  configuration.setString(
      CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, parentFirst);
}
 
Example 8
Source Project: flink   File: YarnLogConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static String getLoggingYarnCommand(final Configuration configuration) {
	checkNotNull(configuration);

	final String logConfigFilePath = configuration.getString(YarnConfigOptionsInternal.APPLICATION_LOG_CONFIG_FILE);
	if (logConfigFilePath == null) {
		return "";
	}

	String logCommand = getLog4jCommand(logConfigFilePath);
	if (logCommand.isEmpty()) {
		logCommand = getLogBackCommand(logConfigFilePath);
	}
	return logCommand;
}
 
Example 9
/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 10
/**
 * Create a security configuration from the global configuration.
 * @param flinkConf the Flink global configuration.
 * @param securityModuleFactories the security modules to apply.
 */
public SecurityConfiguration(Configuration flinkConf,
		List<SecurityModuleFactory> securityModuleFactories) {
	this.isZkSaslDisable = flinkConf.getBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE);
	this.keytab = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
	this.principal = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
	this.useTicketCache = flinkConf.getBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE);
	this.loginContextNames = parseList(flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_CONTEXTS));
	this.zkServiceName = flinkConf.getString(SecurityOptions.ZOOKEEPER_SASL_SERVICE_NAME);
	this.zkLoginContextName = flinkConf.getString(SecurityOptions.ZOOKEEPER_SASL_LOGIN_CONTEXT_NAME);
	this.securityModuleFactories = Collections.unmodifiableList(securityModuleFactories);
	this.flinkConfig = checkNotNull(flinkConf);
	validate();
}
 
Example 11
/**
 * Configures this input format by reading the path to the file from the configuration and the string that
 * defines the record delimiter.
 * 
 * @param parameters The configuration object to read the parameters from.
 */
@Override
public void configure(Configuration parameters) {
	super.configure(parameters);

	// the if() clauses are to prevent the configure() method from
	// overwriting the values set by the setters

	if (Arrays.equals(delimiter, new byte[] {'\n'})) {
		String delimString = parameters.getString(RECORD_DELIMITER, null);
		if (delimString != null) {
			setDelimiter(delimString);
		}
	}
	
	// set the number of samples
	if (numLineSamples == NUM_SAMPLES_UNDEFINED) {
		String samplesString = parameters.getString(NUM_STATISTICS_SAMPLES, null);
		if (samplesString != null) {
			try {
				setNumLineSamples(Integer.parseInt(samplesString));
			} catch (NumberFormatException e) {
				if (LOG.isWarnEnabled()) {
					LOG.warn("Invalid value for number of samples to take: " + samplesString + ". Skipping sampling.");
				}
				setNumLineSamples(0);
			}
		}
	}
}
 
Example 12
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPythonBufferMemorySize() {
	final Configuration configuration = new Configuration();
	final String defaultPythonBufferMemorySize = configuration.getString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE);
	assertThat(defaultPythonBufferMemorySize, is(equalTo(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE.defaultValue())));

	final String expectedPythonBufferMemorySize = "100mb";
	configuration.setString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE, expectedPythonBufferMemorySize);

	final String actualPythonBufferMemorySize = configuration.getString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE);
	assertThat(actualPythonBufferMemorySize, is(equalTo(expectedPythonBufferMemorySize)));
}
 
Example 13
/**
 * Calculates the heap size via
 * {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} and the shell script
 * and verifies that these are equal.
 *
 * @param config     flink configuration
 * @param tolerance  tolerate values that are off by this factor (0.01 = 1%)
 */
private void compareHeapSizeJavaVsScript(final Configuration config, float tolerance)
		throws IOException {

	final long totalJavaMemorySizeMB = config.getLong(KEY_TASKM_MEM_SIZE, 0L);

	long javaHeapSizeMB = TaskManagerServices.calculateHeapSizeMB(totalJavaMemorySizeMB, config);

	String[] command = {"src/test/bin/calcTMHeapSizeMB.sh",
		totalJavaMemorySizeMB + "m",
		String.valueOf(config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)),
		String.valueOf(config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION)),
		config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN),
		config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX),
		config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE),
		String.valueOf(config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION))};
	String scriptOutput = executeScript(command);

	// we need a tolerance of at least one, to compensate for MB/byte conversion rounding errors
	long absoluteTolerance = Math.max(1L, (long) (javaHeapSizeMB * tolerance));

	Long scriptHeapSizeMB = Long.valueOf(scriptOutput);
	assertThat(
		"Different heap sizes (Java: " + javaHeapSizeMB + ", Script: " + scriptHeapSizeMB +
			") with configuration: " + config.toString(), scriptHeapSizeMB,
		allOf(greaterThanOrEqualTo(javaHeapSizeMB - absoluteTolerance),
			lessThanOrEqualTo(javaHeapSizeMB + absoluteTolerance)));
}
 
Example 14
Source Project: flink   File: CliFrontendTestUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void checkJobManagerAddress(Configuration config, String expectedAddress, int expectedPort) {
	String jobManagerAddress = config.getString(JobManagerOptions.ADDRESS);
	int jobManagerPort = config.getInteger(JobManagerOptions.PORT, -1);

	assertEquals(expectedAddress, jobManagerAddress);
	assertEquals(expectedPort, jobManagerPort);
}
 
Example 15
public ZooKeeperRunningJobsRegistry(final CuratorFramework client, final Configuration configuration) {
	this.client = checkNotNull(client, "client");
	this.runningJobPath = configuration.getString(HighAvailabilityOptions.ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH);
}
 
Example 16
public ZooKeeperRunningJobsRegistry(final CuratorFramework client, final Configuration configuration) {
	this.client = checkNotNull(client, "client");
	this.runningJobPath = configuration.getString(HighAvailabilityOptions.ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH);
}
 
Example 17
Source Project: flink   File: BlobServerRecoveryTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Helper to test that the {@link BlobServer} recovery from its HA store works.
 *
 * <p>Uploads two BLOBs to one {@link BlobServer} and expects a second one to be able to retrieve
 * them via a shared HA store upon request of a {@link BlobCacheService}.
 *
 * @param config
 * 		blob server configuration (including HA settings like {@link HighAvailabilityOptions#HA_STORAGE_PATH}
 * 		and {@link HighAvailabilityOptions#HA_CLUSTER_ID}) used to set up <tt>blobStore</tt>
 * @param blobStore
 * 		shared HA blob store to use
 *
 * @throws IOException
 * 		in case of failures
 */
public static void testBlobServerRecovery(final Configuration config, final BlobStore blobStore) throws IOException {
	final String clusterId = config.getString(HighAvailabilityOptions.HA_CLUSTER_ID);
	String storagePath = config.getString(HighAvailabilityOptions.HA_STORAGE_PATH) + "/" + clusterId;
	Random rand = new Random();

	try (
		BlobServer server0 = new BlobServer(config, blobStore);
		BlobServer server1 = new BlobServer(config, blobStore);
		// use VoidBlobStore as the HA store to force download from server[1]'s HA store
		BlobCacheService cache1 = new BlobCacheService(
			config, new VoidBlobStore(), new InetSocketAddress("localhost", server1.getPort())
		)) {

		server0.start();
		server1.start();

		// Random data
		byte[] expected = new byte[1024];
		rand.nextBytes(expected);
		byte[] expected2 = Arrays.copyOfRange(expected, 32, 288);

		BlobKey[] keys = new BlobKey[2];
		BlobKey nonHAKey;

		// Put job-related HA data
		JobID[] jobId = new JobID[] { new JobID(), new JobID() };
		keys[0] = put(server0, jobId[0], expected, PERMANENT_BLOB); // Request 1
		keys[1] = put(server0, jobId[1], expected2, PERMANENT_BLOB); // Request 2

		// put non-HA data
		nonHAKey = put(server0, jobId[0], expected2, TRANSIENT_BLOB);
		verifyKeyDifferentHashEquals(keys[1], nonHAKey);

		// check that the storage directory exists
		final Path blobServerPath = new Path(storagePath, "blob");
		FileSystem fs = blobServerPath.getFileSystem();
		assertTrue("Unknown storage dir: " + blobServerPath, fs.exists(blobServerPath));

		// Verify HA requests from cache1 (connected to server1) with no immediate access to the file
		verifyContents(cache1, jobId[0], keys[0], expected);
		verifyContents(cache1, jobId[1], keys[1], expected2);

		// Verify non-HA file is not accessible from server1
		verifyDeleted(cache1, jobId[0], nonHAKey);

		// Remove again
		server1.cleanupJob(jobId[0], true);
		server1.cleanupJob(jobId[1], true);

		// Verify everything is clean
		assertTrue("HA storage directory does not exist", fs.exists(new Path(storagePath)));
		if (fs.exists(blobServerPath)) {
			final org.apache.flink.core.fs.FileStatus[] recoveryFiles =
				fs.listStatus(blobServerPath);
			ArrayList<String> filenames = new ArrayList<>(recoveryFiles.length);
			for (org.apache.flink.core.fs.FileStatus file: recoveryFiles) {
				filenames.add(file.toString());
			}
			fail("Unclean state backend: " + filenames);
		}
	}
}
 
Example 18
Source Project: Flink-CEPplus   File: SSLUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates an SSL context for the external REST SSL.
 * If mutual authentication is configured the client and the server side configuration are identical.
 */
@Nullable
private static SSLContext createRestSSLContext(Configuration config, RestSSLContextConfigMode configMode) throws Exception {
	checkNotNull(config, "config");

	if (!isRestSSLEnabled(config)) {
		return null;
	}

	KeyManager[] keyManagers = null;
	if (configMode == RestSSLContextConfigMode.SERVER || configMode == RestSSLContextConfigMode.MUTUAL) {
		String keystoreFilePath = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEYSTORE, SecurityOptions.SSL_KEYSTORE);

		String keystorePassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEYSTORE_PASSWORD, SecurityOptions.SSL_KEYSTORE_PASSWORD);

		String certPassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEY_PASSWORD, SecurityOptions.SSL_KEY_PASSWORD);

		KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
		try (InputStream keyStoreFile = Files.newInputStream(new File(keystoreFilePath).toPath())) {
			keyStore.load(keyStoreFile, keystorePassword.toCharArray());
		}

		KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
		kmf.init(keyStore, certPassword.toCharArray());

		keyManagers = kmf.getKeyManagers();
	}

	TrustManager[] trustManagers = null;
	if (configMode == RestSSLContextConfigMode.CLIENT || configMode == RestSSLContextConfigMode.MUTUAL) {
		String trustStoreFilePath = getAndCheckOption(
			config, SecurityOptions.SSL_REST_TRUSTSTORE, SecurityOptions.SSL_TRUSTSTORE);

		String trustStorePassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_TRUSTSTORE_PASSWORD, SecurityOptions.SSL_TRUSTSTORE_PASSWORD);

		KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
		try (InputStream trustStoreFile = Files.newInputStream(new File(trustStoreFilePath).toPath())) {
			trustStore.load(trustStoreFile, trustStorePassword.toCharArray());
		}

		TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
		tmf.init(trustStore);

		trustManagers = tmf.getTrustManagers();
	}

	String sslProtocolVersion = config.getString(SecurityOptions.SSL_PROTOCOL);
	SSLContext sslContext = SSLContext.getInstance(sslProtocolVersion);
	sslContext.init(keyManagers, trustManagers, null);

	return sslContext;
}
 
Example 19
Source Project: Flink-CEPplus   File: SingleInputNode.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
		throws CompilerException
{
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	final ShipStrategyType preSet;
	
	if (shipStrategy != null) {
		if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
			preSet = ShipStrategyType.PARTITION_HASH;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
			preSet = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
			preSet = ShipStrategyType.FORWARD;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
		}
	} else {
		preSet = null;
	}
	
	// get the predecessor node
	Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
	
	OptimizerNode pred;
	DagConnection conn;
	if (children == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
	} else {
		pred = contractToNode.get(children);
		conn = new DagConnection(pred, this, defaultExchangeMode);
		if (preSet != null) {
			conn.setShipStrategy(preSet);
		}
	}
	
	// create the connection and add it
	setIncomingConnection(conn);
	pred.addOutgoingConnection(conn);
}
 
Example 20
Source Project: Flink-CEPplus   File: ClusterEntrypoint.java    License: Apache License 2.0 3 votes vote down vote up
private Configuration generateClusterConfiguration(Configuration configuration) {
	final Configuration resultConfiguration = new Configuration(Preconditions.checkNotNull(configuration));

	final String webTmpDir = configuration.getString(WebOptions.TMP_DIR);
	final File uniqueWebTmpDir = new File(webTmpDir, "flink-web-" + UUID.randomUUID());

	resultConfiguration.setString(WebOptions.TMP_DIR, uniqueWebTmpDir.getAbsolutePath());

	return resultConfiguration;
}