Java Code Examples for org.apache.flink.configuration.Configuration#getInteger()

The following examples show how to use org.apache.flink.configuration.Configuration#getInteger() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public AbstractClientProvider(Configuration properties) {
	this.stsRoleArn = properties.getString(BlinkOptions.STS.STS_ROLE_ARN);
	this.stsAccessId = properties.getString(BlinkOptions.STS.STS_ACCESS_ID);
	this.stsAccessKey = properties.getString(BlinkOptions.STS.STS_ACCESS_KEY);
	this.stsAssumeRoleFor = properties.getString(BlinkOptions.STS.STS_UID);
	this.stsSessionName = String.valueOf(System.currentTimeMillis());
	this.stsExpireSeconds = properties.getInteger(BlinkOptions.STS.STS_ROLEARN_UPDATE_SECONDS);
	useSts = true;
	synchronized (mutex) {
		if (this.cacheArnResponse == null) {
			this.cacheArnResponse = CacheBuilder.newBuilder().concurrencyLevel(5).initialCapacity(1).maximumSize(3)
												.expireAfterWrite(stsExpireSeconds, TimeUnit.SECONDS).build(
							new StsIdentityLoader(this, properties));
		}
	}
}
 
Example 2
Source Project: flink   File: CliFrontend.java    License: Apache License 2.0 6 votes vote down vote up
public CliFrontend(
		Configuration configuration,
		ClusterClientServiceLoader clusterClientServiceLoader,
		List<CustomCommandLine> customCommandLines) {
	this.configuration = checkNotNull(configuration);
	this.customCommandLines = checkNotNull(customCommandLines);
	this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader);

	FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

	this.customCommandLineOptions = new Options();

	for (CustomCommandLine customCommandLine : customCommandLines) {
		customCommandLine.addGeneralOptions(customCommandLineOptions);
		customCommandLine.addRunOptions(customCommandLineOptions);
	}

	this.clientTimeout = configuration.get(ClientOptions.CLIENT_TIMEOUT);
	this.defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
}
 
Example 3
/**
 * Factory method to startWith a new instance, providing a new instance of {@link MiniCluster}
 *
 * @param parallelism global setting for parallel execution.
 * @return new instance of {@link DataStreamTestEnvironment}
 * @throws Exception
 */
public static DataStreamTestEnvironment createTestEnvironment(int parallelism) {
	int taskSlots = Runtime.getRuntime().availableProcessors();

	Configuration configuration = new Configuration();
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskSlots);

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
			.build();

	MiniCluster miniCluster = new MiniCluster(cfg);

	return new DataStreamTestEnvironment(
			miniCluster,
			parallelism);
}
 
Example 4
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example 5
Source Project: flink   File: ClusterEntrypointUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets and verify the io-executor pool size based on configuration.
 *
 * @param config The configuration to read.
 * @return The legal io-executor pool size.
 */
public static int getPoolSize(Configuration config) {
	final int poolSize = config.getInteger(ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE, 4 * Hardware.getNumberCPUCores());
	Preconditions.checkArgument(poolSize > 0,
		String.format("Illegal pool size (%s) of io-executor, please re-configure '%s'.",
			poolSize, ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE.key()));
	return poolSize;
}
 
Example 6
Source Project: flink   File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 
Example 7
public static FailureRateRestartStrategyFactory createFactory(Configuration configuration) throws Exception {
	int maxFailuresPerInterval = configuration.getInteger(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL, 1);
	String failuresIntervalString = configuration.getString(
			ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL, Duration.apply(1, TimeUnit.MINUTES).toString()
	);
	String timeoutString = configuration.getString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL);
	String delayString = configuration.getString(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_DELAY, timeoutString);

	Duration failuresInterval = Duration.apply(failuresIntervalString);
	Duration delay = Duration.apply(delayString);


	return new FailureRateRestartStrategyFactory(maxFailuresPerInterval, Time.milliseconds(failuresInterval.toMillis()), Time.milliseconds(delay.toMillis()));
}
 
Example 8
/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 
Example 9
Source Project: flink   File: DistributedCache.java    License: Apache License 2.0 5 votes vote down vote up
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
 
Example 10
Source Project: flink   File: DelimitedInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
protected static void loadConfigParameters(Configuration parameters) {
	int maxSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES);
	int minSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES);
	
	if (maxSamples < 0) {
		LOG.error("Invalid default maximum number of line samples: " + maxSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key());
		maxSamples = OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.defaultValue();
	}
	if (minSamples < 0) {
		LOG.error("Invalid default minimum number of line samples: " + minSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key());
		minSamples = OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.defaultValue();
	}
	
	DEFAULT_MAX_NUM_SAMPLES = maxSamples;
	
	if (minSamples > maxSamples) {
		LOG.error("Default minimum number of line samples cannot be greater the default maximum number " +
				"of line samples: min=" + minSamples + ", max=" + maxSamples + ". Defaulting minimum to maximum.");
		DEFAULT_MIN_NUM_SAMPLES = maxSamples;
	} else {
		DEFAULT_MIN_NUM_SAMPLES = minSamples;
	}
	
	int maxLen = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN);
	if (maxLen <= 0) {
		maxLen = OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue();
		LOG.error("Invalid value for the maximum sample record length. Using default value of " + maxLen + '.');
	} else if (maxLen < DEFAULT_READ_BUFFER_SIZE) {
		maxLen = DEFAULT_READ_BUFFER_SIZE;
		LOG.warn("Increasing maximum sample record length to size of the read buffer (" + maxLen + ").");
	}
	MAX_SAMPLE_LEN = maxLen;
}
 
Example 11
/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataport(Configuration configuration) {
	final int dataport = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
	ConfigurationParserUtils.checkConfigParameter(dataport >= 0, dataport, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
		"Leave config parameter empty or use 0 to let the system choose a port automatically.");

	return dataport;
}
 
Example 12
Source Project: flink   File: MesosServicesUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static MesosArtifactServer createArtifactServer(Configuration configuration, String hostname) throws Exception {
	final int artifactServerPort = configuration.getInteger(MesosOptions.ARTIFACT_SERVER_PORT, 0);

	// a random prefix is affixed to artifact URLs to ensure uniqueness in the Mesos fetcher cache
	final String artifactServerPrefix = UUID.randomUUID().toString();

	return new MesosArtifactServer(artifactServerPrefix, hostname, artifactServerPort, configuration);
}
 
Example 13
Source Project: flink   File: ClusterSpecification.java    License: Apache License 2.0 5 votes vote down vote up
public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 
Example 14
/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 * specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	// transform the streaming program into a JobGraph
	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}
 
Example 15
@Override
public OptionsFactory configure(Configuration configuration) {
	this.backgroundJobs = configuration.getInteger(BACKGROUND_JOBS_OPTION, DEFAULT_BACKGROUND_JOBS);
	return this;
}
 
Example 16
Source Project: flink   File: MapITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(Configuration config) {
	int val = config.getInteger(TEST_KEY, -1);
	Assert.assertEquals(TEST_VALUE, val);
}
 
Example 17
/**
 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST server endpoint configuration should be created from
 * @return REST server endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */
public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.ADDRESS),
		"%s must be set",
		RestOptions.ADDRESS.key());

	final String restBindAddress = config.getString(RestOptions.BIND_ADDRESS);
	final String portRangeDefinition = config.getString(RestOptions.BIND_PORT);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestServerSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLEngineFactory for REST server endpoint.", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final Path uploadDir = Paths.get(
		config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
		"flink-web-upload");

	final int maxContentLength = config.getInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH);

	final Map<String, String> responseHeaders = Collections.singletonMap(
		HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN,
		config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN));

	return new RestServerEndpointConfiguration(
		restAddress,
		restBindAddress,
		portRangeDefinition,
		sslHandlerFactory,
		uploadDir,
		maxContentLength,
		responseHeaders);
}
 
Example 18
Source Project: flink   File: NettyClientServerSslTest.java    License: Apache License 2.0 4 votes vote down vote up
private void testValidSslConnection(Configuration sslConfig) throws Exception {
	OneShotLatch serverChannelInitComplete = new OneShotLatch();
	final SslHandler[] serverSslHandler = new SslHandler[1];

	NettyProtocol protocol = new NoOpProtocol();

	NettyConfig nettyConfig = createNettyConfig(sslConfig);

	final NettyBufferPool bufferPool = new NettyBufferPool(1);
	final NettyServer server = NettyTestUtil.initServer(
		nettyConfig,
		bufferPool,
		sslHandlerFactory ->
			new TestingServerChannelInitializer(
				protocol,
				sslHandlerFactory,
				serverChannelInitComplete,
				serverSslHandler));
	final NettyClient client = NettyTestUtil.initClient(nettyConfig, protocol, bufferPool);
	final NettyServerAndClient serverAndClient = new NettyServerAndClient(server, client);

	Channel ch = NettyTestUtil.connect(serverAndClient);

	SslHandler clientSslHandler = (SslHandler) ch.pipeline().get("ssl");
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_HANDSHAKE_TIMEOUT, clientSslHandler.getHandshakeTimeoutMillis());
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT, clientSslHandler.getCloseNotifyFlushTimeoutMillis());

	// should be able to send text data
	ch.pipeline().addLast(new StringDecoder()).addLast(new StringEncoder());
	ch.writeAndFlush("test").sync();

	// session context is only be available after a session was setup -> this should be true after data was sent
	serverChannelInitComplete.await();
	assertNotNull(serverSslHandler[0]);

	// verify server parameters
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_HANDSHAKE_TIMEOUT, serverSslHandler[0].getHandshakeTimeoutMillis());
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT, serverSslHandler[0].getCloseNotifyFlushTimeoutMillis());
	SSLSessionContext sessionContext = serverSslHandler[0].engine().getSession().getSessionContext();
	assertNotNull("bug in unit test setup: session context not available", sessionContext);
	// note: can't verify session cache setting at the client - delegate to server instead (with our own channel initializer)
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_SESSION_CACHE_SIZE, sessionContext.getSessionCacheSize());
	int sessionTimeout = sslConfig.getInteger(SSL_INTERNAL_SESSION_TIMEOUT);
	if (sessionTimeout != -1) {
		// session timeout config is in milliseconds but the context returns it in seconds
		assertEquals(sessionTimeout / 1000, sessionContext.getSessionTimeout());
	} else {
		assertTrue("default value (-1) should not be propagated", sessionContext.getSessionTimeout() >= 0);
	}

	NettyTestUtil.shutdown(serverAndClient);
}
 
Example 19
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example 20
Source Project: toolbox   File: ConversionToBatches.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    batchSize = parameters.getInteger(BATCH_SIZE, 1000);
    attributes = Serialization.deserializeObject(parameters.getBytes(ATTRIBUTES, null));
}