Java Code Examples for org.apache.flink.configuration.ConfigurationUtils#parseTempDirectories()

The following examples show how to use org.apache.flink.configuration.ConfigurationUtils#parseTempDirectories() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JobMasterConfiguration.java From Flink-CEPplus with Apache License 2.0

6 votes

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}

Example 2

Source File: SessionClusterEntrypoint.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}

Example 3

Source File: JobMasterConfiguration.java From flink with Apache License 2.0

6 votes

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}

Example 4

Source File: SessionClusterEntrypoint.java From flink with Apache License 2.0

6 votes

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}

Example 5

Source File: SavepointEnvironment.java From flink with Apache License 2.0

6 votes

private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) {
	this.jobID = new JobID();
	this.vertexID = new JobVertexID();
	this.attemptID = new ExecutionAttemptID();
	this.ctx = Preconditions.checkNotNull(ctx);
	this.configuration = Preconditions.checkNotNull(configuration);

	Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism);
	this.maxParallelism = maxParallelism;
	this.indexOfSubtask = indexOfSubtask;

	this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID);
	this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState);
	this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration));
	this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024);
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID);
}

Example 6

Source File: JobMasterConfiguration.java From flink with Apache License 2.0

6 votes

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}

Example 7

Source File: SessionClusterEntrypoint.java From flink with Apache License 2.0

6 votes

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY);
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCapacity,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}

Example 8

Source File: TaskManagerServicesConfiguration.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);

	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(
		CheckpointingOptions.LOCAL_RECOVERY.key(),
		CheckpointingOptions.LOCAL_RECOVERY.defaultValue());

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			configuredMemory = MemorySize.parse(configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		configuredMemory = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(
		configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}

Example 9

Source File: NettyShuffleEnvironmentConfiguration.java From flink with Apache License 2.0

4 votes

/**
 * Utility method to extract network related parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration configuration object
 * @param maxJvmHeapMemory the maximum JVM heap size (in bytes)
 * @param localTaskManagerCommunication true, to skip initializing the network stack
 * @param taskManagerAddress identifying the IP address under which the TaskManager will be accessible
 * @return NettyShuffleEnvironmentConfiguration
 */
public static NettyShuffleEnvironmentConfiguration fromConfiguration(
	Configuration configuration,
	long maxJvmHeapMemory,
	boolean localTaskManagerCommunication,
	InetAddress taskManagerAddress) {

	final int dataport = getDataport(configuration);

	final int pageSize = ConfigurationParserUtils.getPageSize(configuration);

	final int numberOfNetworkBuffers = calculateNumberOfNetworkBuffers(configuration, maxJvmHeapMemory);

	final NettyConfig nettyConfig = createNettyConfig(configuration, localTaskManagerCommunication, taskManagerAddress, dataport);

	int initialRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
	int maxRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX);

	int buffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL);
	int extraBuffersPerGate = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);

	boolean isCreditBased = nettyConfig != null && configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_CREDIT_MODEL);

	boolean isNetworkDetailedMetrics = configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_DETAILED_METRICS);

	String[] tempDirs = ConfigurationUtils.parseTempDirectories(configuration);

	Duration requestSegmentsTimeout = Duration.ofMillis(configuration.getLong(
			NettyShuffleEnvironmentOptions.NETWORK_EXCLUSIVE_BUFFERS_REQUEST_TIMEOUT_MILLISECONDS));

	BoundedBlockingSubpartitionType blockingSubpartitionType = getBlockingSubpartitionType(configuration);

	boolean forcePartitionReleaseOnConsumption =
		configuration.getBoolean(NettyShuffleEnvironmentOptions.FORCE_PARTITION_RELEASE_ON_CONSUMPTION);

	return new NettyShuffleEnvironmentConfiguration(
		numberOfNetworkBuffers,
		pageSize,
		initialRequestBackoff,
		maxRequestBackoff,
		buffersPerChannel,
		extraBuffersPerGate,
		requestSegmentsTimeout,
		isCreditBased,
		isNetworkDetailedMetrics,
		nettyConfig,
		tempDirs,
		blockingSubpartitionType,
		forcePartitionReleaseOnConsumption);
}

Example 10

Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0

4 votes

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param freeHeapMemoryWithDefrag an estimate of the size of the free heap memory
 * @param maxJvmHeapMemory the maximum JVM heap size
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		InetAddress remoteAddress,
		long freeHeapMemoryWithDefrag,
		long maxJvmHeapMemory,
		boolean localCommunicationOnly) {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		remoteAddress,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		freeHeapMemoryWithDefrag,
		maxJvmHeapMemory,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getManagedMemorySize(configuration),
		ConfigurationParserUtils.getMemoryType(configuration),
		preAllocateMemory,
		ConfigurationParserUtils.getManagedMemoryFraction(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}

Example 11

Source File: NettyShuffleEnvironmentConfiguration.java From flink with Apache License 2.0

4 votes

/**
 * Utility method to extract network related parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration configuration object
 * @param networkMemorySize the size of memory reserved for shuffle environment
 * @param localTaskManagerCommunication true, to skip initializing the network stack
 * @param taskManagerAddress identifying the IP address under which the TaskManager will be accessible
 * @return NettyShuffleEnvironmentConfiguration
 */
public static NettyShuffleEnvironmentConfiguration fromConfiguration(
	Configuration configuration,
	MemorySize networkMemorySize,
	boolean localTaskManagerCommunication,
	InetAddress taskManagerAddress) {

	final int dataBindPort = getDataBindPort(configuration);

	final int pageSize = ConfigurationParserUtils.getPageSize(configuration);

	final NettyConfig nettyConfig = createNettyConfig(configuration, localTaskManagerCommunication, taskManagerAddress, dataBindPort);

	final int numberOfNetworkBuffers = calculateNumberOfNetworkBuffers(
		configuration,
		networkMemorySize,
		pageSize);

	int initialRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
	int maxRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX);

	int buffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL);
	int extraBuffersPerGate = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);

	int maxBuffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_MAX_BUFFERS_PER_CHANNEL);

	boolean isNetworkDetailedMetrics = configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_DETAILED_METRICS);

	String[] tempDirs = ConfigurationUtils.parseTempDirectories(configuration);

	Duration requestSegmentsTimeout = Duration.ofMillis(configuration.getLong(
			NettyShuffleEnvironmentOptions.NETWORK_EXCLUSIVE_BUFFERS_REQUEST_TIMEOUT_MILLISECONDS));

	BoundedBlockingSubpartitionType blockingSubpartitionType = getBlockingSubpartitionType(configuration);

	boolean forcePartitionReleaseOnConsumption =
		configuration.getBoolean(NettyShuffleEnvironmentOptions.FORCE_PARTITION_RELEASE_ON_CONSUMPTION);

	boolean blockingShuffleCompressionEnabled =
		configuration.get(NettyShuffleEnvironmentOptions.BLOCKING_SHUFFLE_COMPRESSION_ENABLED);
	String compressionCodec = configuration.getString(NettyShuffleEnvironmentOptions.SHUFFLE_COMPRESSION_CODEC);

	return new NettyShuffleEnvironmentConfiguration(
		numberOfNetworkBuffers,
		pageSize,
		initialRequestBackoff,
		maxRequestBackoff,
		buffersPerChannel,
		extraBuffersPerGate,
		requestSegmentsTimeout,
		isNetworkDetailedMetrics,
		nettyConfig,
		tempDirs,
		blockingSubpartitionType,
		forcePartitionReleaseOnConsumption,
		blockingShuffleCompressionEnabled,
		compressionCodec,
		maxBuffersPerChannel);
}

Example 12

Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0

4 votes

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}