Java Code Examples for org.apache.flink.configuration.Configuration#getBoolean()

The following examples show how to use org.apache.flink.configuration.Configuration#getBoolean() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Configures the file input format by reading the file path from the configuration.
 * 
 * @see org.apache.flink.api.common.io.InputFormat#configure(org.apache.flink.configuration.Configuration)
 */
@Override
public void configure(Configuration parameters) {

	if (getFilePaths().length == 0) {
		// file path was not specified yet. Try to set it from the parameters.
		String filePath = parameters.getString(FILE_PARAMETER_KEY, null);
		if (filePath == null) {
			throw new IllegalArgumentException("File path was not specified in input format or configuration.");
		} else {
			setFilePath(filePath);
		}
	}

	if (!this.enumerateNestedFiles) {
		this.enumerateNestedFiles = parameters.getBoolean(ENUMERATE_NESTED_FILES_FLAG, false);
	}
}
 
Example 2
Source File: QueryableStateConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 */
public static QueryableStateConfiguration fromConfiguration(Configuration config) {
	if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) {
		return null;
	}

	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.PROXY_PORT_RANGE));
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.SERVER_PORT_RANGE));

	final int numProxyServerNetworkThreads = config.getInteger(QueryableStateOptions.PROXY_NETWORK_THREADS);
	final int numProxyServerQueryThreads = config.getInteger(QueryableStateOptions.PROXY_ASYNC_QUERY_THREADS);

	final int numStateServerNetworkThreads = config.getInteger(QueryableStateOptions.SERVER_NETWORK_THREADS);
	final int numStateServerQueryThreads = config.getInteger(QueryableStateOptions.SERVER_ASYNC_QUERY_THREADS);

	return new QueryableStateConfiguration(
		proxyPorts,
		serverPorts,
		numProxyServerNetworkThreads,
		numProxyServerQueryThreads,
		numStateServerNetworkThreads,
		numStateServerQueryThreads);
}
 
Example 3
Source File: DefaultSchedulerFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static SlotSelectionStrategy selectSlotSelectionStrategy(@Nonnull Configuration configuration) {
	final boolean evenlySpreadOutSlots = configuration.getBoolean(ClusterOptions.EVENLY_SPREAD_OUT_SLOTS_STRATEGY);

	final SlotSelectionStrategy locationPreferenceSlotSelectionStrategy;

	if (evenlySpreadOutSlots) {
		locationPreferenceSlotSelectionStrategy = LocationPreferenceSlotSelectionStrategy.createEvenlySpreadOut();
	} else {
		locationPreferenceSlotSelectionStrategy = LocationPreferenceSlotSelectionStrategy.createDefault();
	}

	if (configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY)) {
		return PreviousAllocationSlotSelectionStrategy.create(locationPreferenceSlotSelectionStrategy);
	} else {
		return locationPreferenceSlotSelectionStrategy;
	}
}
 
Example 4
Source File: TaskManagerServicesConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 */
private static QueryableStateConfiguration parseQueryableStateConfiguration(Configuration config) {
	if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) {
		return null;
	}

	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(
			config.getString(QueryableStateOptions.PROXY_PORT_RANGE));
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(
			config.getString(QueryableStateOptions.SERVER_PORT_RANGE));

	final int numProxyServerNetworkThreads = config.getInteger(QueryableStateOptions.PROXY_NETWORK_THREADS);
	final int numProxyServerQueryThreads = config.getInteger(QueryableStateOptions.PROXY_ASYNC_QUERY_THREADS);

	final int numStateServerNetworkThreads = config.getInteger(QueryableStateOptions.SERVER_NETWORK_THREADS);
	final int numStateServerQueryThreads = config.getInteger(QueryableStateOptions.SERVER_ASYNC_QUERY_THREADS);

	return new QueryableStateConfiguration(
			proxyPorts,
			serverPorts,
			numProxyServerNetworkThreads,
			numProxyServerQueryThreads,
			numStateServerNetworkThreads,
			numStateServerQueryThreads);
}
 
Example 5
Source File: AkkaRpcServiceUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example 6
Source File: SlotManagerConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static SlotManagerConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
	final String strTimeout = configuration.getString(AkkaOptions.ASK_TIMEOUT);
	final Time rpcTimeout;

	try {
		rpcTimeout = Time.milliseconds(Duration.apply(strTimeout).toMillis());
	} catch (NumberFormatException e) {
		throw new ConfigurationException("Could not parse the resource manager's timeout " +
			"value " + AkkaOptions.ASK_TIMEOUT + '.', e);
	}

	final Time slotRequestTimeout = getSlotRequestTimeout(configuration);
	final Time taskManagerTimeout = Time.milliseconds(
			configuration.getLong(ResourceManagerOptions.TASK_MANAGER_TIMEOUT));

	boolean waitResultConsumedBeforeRelease =
		configuration.getBoolean(ResourceManagerOptions.TASK_MANAGER_RELEASE_WHEN_RESULT_CONSUMED);

	return new SlotManagerConfiguration(rpcTimeout, slotRequestTimeout, taskManagerTimeout, waitResultConsumedBeforeRelease);
}
 
Example 7
Source File: ConfigurationParserUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Parses the configuration to get the type of memory.
 *
 * @param configuration configuration object
 * @return type of memory
 */
public static MemoryType getMemoryType(Configuration configuration) {
	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}
	return memType;
}
 
Example 8
Source File: ParquetInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void configure(Configuration parameters) {
	super.configure(parameters);

	if (!this.skipWrongSchemaFileSplit) {
		this.skipWrongSchemaFileSplit = parameters.getBoolean(PARQUET_SKIP_WRONG_SCHEMA_SPLITS, false);
	}

	if (this.skipCorruptedRecord) {
		this.skipCorruptedRecord = parameters.getBoolean(PARQUET_SKIP_CORRUPTED_RECORD, false);
	}
}
 
Example 9
Source File: RecordComparatorFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void readParametersFromConfig(Configuration config, ClassLoader cl) throws ClassNotFoundException {
	// figure out how many key fields there are
	final int numKeyFields = config.getInteger(NUM_KEYS, -1);
	if (numKeyFields < 0) {
		throw new IllegalConfigurationException("The number of keys for the comparator is invalid: " + numKeyFields);
	}
	
	final int[] positions = new int[numKeyFields];
	final Class<? extends Value>[] types = new Class[numKeyFields];
	final boolean[] direction = new boolean[numKeyFields];
	
	// read the individual key positions and types
	for (int i = 0; i < numKeyFields; i++) {
		// next key position
		final int p = config.getInteger(KEY_POS_PREFIX + i, -1);
		if (p >= 0) {
			positions[i] = p;
		} else {
			throw new IllegalConfigurationException("Contained invalid position for key no positions for keys.");
		}
		
		// next key type
		final String name = config.getString(KEY_CLASS_PREFIX + i, null);
		if (name != null) {
			types[i] = (Class<? extends Value>) Class.forName(name, true, cl).asSubclass(Value.class);
		} else {
			throw new IllegalConfigurationException("The key type (" + i +
				") for the comparator is null"); 
		}
		
		// next key sort direction
		direction[i] = config.getBoolean(KEY_SORT_DIRECTION_PREFIX + i, true);
	}
	
	this.positions = positions;
	this.types = types;
	this.sortDirections = direction;
}
 
Example 10
Source File: BufferedKVExternalSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
public BufferedKVExternalSorter(
		IOManager ioManager,
		BinaryRowSerializer keySerializer,
		BinaryRowSerializer valueSerializer,
		NormalizedKeyComputer nKeyComputer,
		RecordComparator comparator,
		int pageSize,
		Configuration conf) throws IOException {
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;
	this.nKeyComputer = nKeyComputer;
	this.comparator = comparator;
	this.pageSize = pageSize;
	this.sorter = new QuickSort();
	this.maxNumFileHandles = conf.getInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES);
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
		? BlockCompressionFactory.createBlockCompressionFactory(
			BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
		: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();
	this.ioManager = ioManager;
	this.enumerator = this.ioManager.createChannelEnumerator();
	this.channelManager = new SpillChannelManager();
	this.merger = new BinaryKVExternalMerger(
			ioManager, pageSize,
			maxNumFileHandles, channelManager,
			keySerializer, valueSerializer, comparator,
			compressionEnable,
		compressionCodecFactory,
			compressionBlockSize);
}
 
Example 11
Source File: ParquetInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void configure(Configuration parameters) {
	super.configure(parameters);

	if (!this.skipWrongSchemaFileSplit) {
		this.skipWrongSchemaFileSplit = parameters.getBoolean(PARQUET_SKIP_WRONG_SCHEMA_SPLITS, false);
	}

	if (this.skipCorruptedRecord) {
		this.skipCorruptedRecord = parameters.getBoolean(PARQUET_SKIP_CORRUPTED_RECORD, false);
	}
}
 
Example 12
Source File: SummarizationJobParameters.java    From timely with Apache License 2.0 5 votes vote down vote up
public SummarizationJobParameters(Configuration conf) {
    System.out.println("Creating job parameters from configuration: " + conf);
    timelyHostname = conf.getString("timelyHostname", null);
    timelyTcpPort = conf.getInteger("timelyTcpPort", 4241);
    timelyHttpsPort = conf.getInteger("timelyHttpsPort", 4242);
    timelyWssPort = conf.getInteger("timelyWssPort", 4243);
    clientAuth = conf.getBoolean("clientAuth", false);
    doLogin = conf.getBoolean("doLogin", false);
    timelyUsername = conf.getString("timelyUsername", null);
    timelyPassword = conf.getString("timelyPassword", null);
    keyStoreFile = conf.getString("keyStoreFile", null);
    keyStoreType = conf.getString("keyStoreType", "JKS");
    keyStorePass = conf.getString("keyStorePass", null);
    trustStoreFile = conf.getString("trustStoreFile", null);
    trustStoreType = conf.getString("trustStoreType", "JKS");
    trustStorePass = conf.getString("trustStorePass", null);
    hostVerificationEnabled = conf.getBoolean("hostVerificationEnabled", true);
    bufferSize = conf.getInteger("bufferSize", 10485760);
    String metricNames = conf.getString("metrics", null);
    if (null != metricNames) {
        metrics = metricNames.split(",");
    } else {
        metrics = null;
    }
    startTime = conf.getLong("startTime", 0L);
    endTime = conf.getLong("endTime", 0L);
    interval = conf.getString("interval", null);
    intervalUnits = conf.getString("intervalUnits", null);
}
 
Example 13
Source File: MemoryLogger.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void startIfConfigured(
		Logger logger,
		Configuration configuration,
		CompletableFuture<Void> taskManagerTerminationFuture) {
	if (!logger.isInfoEnabled() || !configuration.getBoolean(TaskManagerOptions.DEBUG_MEMORY_LOG)) {
		return;
	}
	logger.info("Starting periodic memory usage logger");

	new MemoryLogger(
		logger,
		configuration.getLong(TaskManagerOptions.DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS),
		taskManagerTerminationFuture).start();
}
 
Example 14
Source File: JobGraphGenerator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 
Example 15
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
 
Example 16
Source File: YarnResourceManager.java    From flink with Apache License 2.0 4 votes vote down vote up
public YarnResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		flinkConfig,
		env,
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup);
	this.yarnConfig = new YarnConfiguration();
	this.workerNodeMap = new ConcurrentHashMap<>();
	final int yarnHeartbeatIntervalMS = flinkConfig.getInteger(
			YarnConfigOptions.HEARTBEAT_DELAY_SECONDS) * 1000;

	final long yarnExpiryIntervalMS = yarnConfig.getLong(
			YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
			YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);

	if (yarnHeartbeatIntervalMS >= yarnExpiryIntervalMS) {
		log.warn("The heartbeat interval of the Flink Application master ({}) is greater " +
				"than YARN's expiry interval ({}). The application is likely to be killed by YARN.",
				yarnHeartbeatIntervalMS, yarnExpiryIntervalMS);
	}
	yarnHeartbeatIntervalMillis = yarnHeartbeatIntervalMS;
	containerRequestHeartbeatIntervalMillis = flinkConfig.getInteger(YarnConfigOptions.CONTAINER_REQUEST_HEARTBEAT_INTERVAL_MILLISECONDS);

	this.webInterfaceUrl = webInterfaceUrl;

	this.workerSpecContainerResourceAdapter = new WorkerSpecContainerResourceAdapter(
		flinkConfig,
		yarnConfig.getInt(
			YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
			YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
		yarnConfig.getInt(
			YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
			YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
		yarnConfig.getInt(
			YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
			YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
		yarnConfig.getInt(
			YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
			YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
		ExternalResourceUtils.getExternalResources(flinkConfig, YarnConfigOptions.EXTERNAL_RESOURCE_YARN_CONFIG_KEY_SUFFIX));
	this.registerApplicationMasterResponseReflector = new RegisterApplicationMasterResponseReflector(log);

	this.matchingStrategy = flinkConfig.getBoolean(YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES) ?
		WorkerSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
		WorkerSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
}
 
Example 17
Source File: BlobServer.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	final String bindHost = config.getOptional(JobManagerOptions.BIND_HOST).orElseGet(NetUtils::getWildcardIPAddress);

	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog, InetAddress.getByName(bindHost)));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 
Example 18
Source File: SSLUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Checks whether mutual SSL authentication for the external REST endpoint is enabled.
 */
public static boolean isRestSSLAuthenticationEnabled(Configuration sslConfig) {
	checkNotNull(sslConfig, "sslConfig");
	return isRestSSLEnabled(sslConfig) &&
		sslConfig.getBoolean(SecurityOptions.SSL_REST_AUTHENTICATION_ENABLED);
}
 
Example 19
Source File: SSLUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Checks whether mutual SSL authentication for the external REST endpoint is enabled.
 */
public static boolean isRestSSLAuthenticationEnabled(Configuration sslConfig) {
	checkNotNull(sslConfig, "sslConfig");
	return isRestSSLEnabled(sslConfig) &&
		sslConfig.getBoolean(SecurityOptions.SSL_REST_AUTHENTICATION_ENABLED);
}
 
Example 20
Source File: FileOutputFormat.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Initialize defaults for output format. Needs to be a static method because it is configured for local
 * cluster execution.
 * @param configuration The configuration to load defaults from
 */
public static void initDefaultsFromConfiguration(Configuration configuration) {
	final boolean overwrite = configuration.getBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE);

	DEFAULT_WRITE_MODE = overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE;
	
	final boolean alwaysCreateDirectory = configuration.getBoolean(CoreOptions.FILESYSTEM_OUTPUT_ALWAYS_CREATE_DIRECTORY);

	DEFAULT_OUTPUT_DIRECTORY_MODE = alwaysCreateDirectory ? OutputDirectoryMode.ALWAYS : OutputDirectoryMode.PARONLY;
}