Java Code Examples for org.apache.flink.configuration.IllegalConfigurationException

The following examples show how to use org.apache.flink.configuration.IllegalConfigurationException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 2
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 3
/**
 * Validates the (new) network buffer configuration.
 *
 * @param pageSize 				size of memory buffers
 * @param networkBufFraction	fraction of JVM memory to use for network buffers
 * @param networkBufMin 		minimum memory size for network buffers (in bytes)
 * @param networkBufMax 		maximum memory size for network buffers (in bytes)
 *
 * @throws IllegalConfigurationException if the condition does not hold
 */
protected static void checkNetworkBufferConfig(
		final int pageSize, final float networkBufFraction, final long networkBufMin,
		final long networkBufMax) throws IllegalConfigurationException {

	checkConfigParameter(networkBufFraction > 0.0f && networkBufFraction < 1.0f, networkBufFraction,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
		"Network buffer memory fraction of the free memory must be between 0.0 and 1.0");

	checkConfigParameter(networkBufMin >= pageSize, networkBufMin,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
		"Minimum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= pageSize, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= networkBufMin, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must not be smaller than minimum memory (" +
			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ": " + networkBufMin + ")");
}
 
Example 4
Source Project: Flink-CEPplus   Source File: FsStateBackendFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: FlinkZooKeeperQuorumPeer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write 'myid' file to the 'dataDir' in the given ZooKeeper configuration.
 *
 * <blockquote>
 * Every machine that is part of the ZooKeeper ensemble should know about every other machine in
 * the ensemble. You accomplish this with the series of lines of the form
 * server.id=host:port:port. The parameters host and port are straightforward. You attribute the
 * server id to each machine by creating a file named myid, one for each server, which resides
 * in that server's data directory, as specified by the configuration file parameter dataDir.
 * </blockquote>
 *
 * @param zkProps ZooKeeper configuration.
 * @param id      The ID of this {@link QuorumPeer}.
 * @throws IllegalConfigurationException Thrown, if 'dataDir' property not set in given
 *                                       ZooKeeper properties.
 * @throws IOException                   Thrown, if 'dataDir' does not exist and cannot be
 *                                       created.
 * @see <a href="http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html">
 * ZooKeeper Administrator's Guide</a>
 */

private static void writeMyIdToDataDir(Properties zkProps, int id) throws IOException {

	// Check dataDir and create if necessary
	if (zkProps.getProperty("dataDir") == null) {
		throw new IllegalConfigurationException("No dataDir configured.");
	}

	File dataDir = new File(zkProps.getProperty("dataDir"));

	if (!dataDir.isDirectory() && !dataDir.mkdirs()) {
		throw new IOException("Cannot create dataDir '" + dataDir + "'.");
	}

	dataDir.deleteOnExit();

	LOG.info("Writing {} to myid file in 'dataDir'.", id);
	
	// Write myid to file. We use a File Writer, because that properly propagates errors,
	// while the PrintWriter swallows errors
	try (FileWriter writer = new FileWriter(new File(dataDir, "myid"))) {
		writer.write(String.valueOf(id));
	}
}
 
Example 6
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a SSLEngineFactory to be used by internal communication server endpoints.
 */
public static SSLHandlerFactory createInternalServerSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for internal communication.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			false,
			true,
			config.getInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT),
			config.getInteger(SecurityOptions.SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT));
}
 
Example 7
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a SSLEngineFactory to be used by internal communication client endpoints.
 */
public static SSLHandlerFactory createInternalClientSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for internal communication.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			true,
			true,
			config.getInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT),
			config.getInteger(SecurityOptions.SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT));
}
 
Example 8
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link SSLHandlerFactory} to be used by the REST Servers.
 *
 * @param config The application configuration.
 */
public static SSLHandlerFactory createRestServerSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createRestServerSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for REST endpoints.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			false,
			isRestSSLAuthenticationEnabled(config),
			-1,
			-1);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link SSLHandlerFactory} to be used by the REST Clients.
 *
 * @param config The application configuration.
 */
public static SSLHandlerFactory createRestClientSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createRestClientSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for REST endpoints.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			true,
			isRestSSLAuthenticationEnabled(config),
			-1,
			-1);
}
 
Example 10
Source Project: flink   Source File: YarnHighAvailabilityServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 
Example 11
Source Project: flink   Source File: TaskManagerServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Gets the size of managed memory from the heap size after subtracting network buffer memory.
 * All values are in bytes.
 */
public static long getManagedMemoryFromHeapAndManaged(Configuration config, long heapAndManagedMemory) {
	if (config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		// take the configured absolute value
		final String sizeValue = config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE);
		try {
			return MemorySize.parse(sizeValue, MEGA_BYTES).getBytes();
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	}
	else {
		// calculate managed memory size via fraction
		final float fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
		return (long) (fraction * heapAndManagedMemory);
	}
}
 
Example 12
Source Project: flink   Source File: AbstractFileStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
private static Path parameterOrConfigured(@Nullable Path path, Configuration config, ConfigOption<String> option) {
	if (path != null) {
		return path;
	}
	else {
		String configValue = config.getString(option);
		try {
			return configValue == null ? null : new Path(configValue);
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Cannot parse value for " + option.key() +
					" : " + configValue + " . Not a valid path.");
		}
	}
}
 
Example 13
Source Project: flink   Source File: FsStateBackendFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 
Example 14
Source Project: flink   Source File: FlinkZooKeeperQuorumPeer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write 'myid' file to the 'dataDir' in the given ZooKeeper configuration.
 *
 * <blockquote>
 * Every machine that is part of the ZooKeeper ensemble should know about every other machine in
 * the ensemble. You accomplish this with the series of lines of the form
 * server.id=host:port:port. The parameters host and port are straightforward. You attribute the
 * server id to each machine by creating a file named myid, one for each server, which resides
 * in that server's data directory, as specified by the configuration file parameter dataDir.
 * </blockquote>
 *
 * @param zkProps ZooKeeper configuration.
 * @param id      The ID of this {@link QuorumPeer}.
 * @throws IllegalConfigurationException Thrown, if 'dataDir' property not set in given
 *                                       ZooKeeper properties.
 * @throws IOException                   Thrown, if 'dataDir' does not exist and cannot be
 *                                       created.
 * @see <a href="http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html">
 * ZooKeeper Administrator's Guide</a>
 */

private static void writeMyIdToDataDir(Properties zkProps, int id) throws IOException {

	// Check dataDir and create if necessary
	if (zkProps.getProperty("dataDir") == null) {
		throw new IllegalConfigurationException("No dataDir configured.");
	}

	File dataDir = new File(zkProps.getProperty("dataDir"));

	if (!dataDir.isDirectory() && !dataDir.mkdirs()) {
		throw new IOException("Cannot create dataDir '" + dataDir + "'.");
	}

	dataDir.deleteOnExit();

	LOG.info("Writing {} to myid file in 'dataDir'.", id);
	
	// Write myid to file. We use a File Writer, because that properly propagates errors,
	// while the PrintWriter swallows errors
	try (FileWriter writer = new FileWriter(new File(dataDir, "myid"))) {
		writer.write(String.valueOf(id));
	}
}
 
Example 15
Source Project: flink   Source File: ConfigurationParserUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses the configuration to get the managed memory size and validates the value.
 *
 * @param configuration configuration object
 * @return managed memory size (in megabytes)
 */
public static long getManagedMemorySize(Configuration configuration) {
	long managedMemorySize;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			managedMemorySize = MemorySize.parse(
				configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		managedMemorySize = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(configuration.getString(
		TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0,
		managedMemorySize, TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically pick a fraction of the available memory.");

	return managedMemorySize;
}
 
Example 16
Source Project: Flink-CEPplus   Source File: NetUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns an address in a normalized format for Akka.
 * When an IPv6 address is specified, it normalizes the IPv6 address to avoid
 * complications with the exact URL match policy of Akka.
 * @param host The hostname, IPv4 or IPv6 address
 * @return host which will be normalized if it is an IPv6 address
 */
public static String unresolvedHostToNormalizedString(String host) {
	// Return loopback interface address if host is null
	// This represents the behavior of {@code InetAddress.getByName } and RFC 3330
	if (host == null) {
		host = InetAddress.getLoopbackAddress().getHostAddress();
	} else {
		host = host.trim().toLowerCase();
	}

	// normalize and valid address
	if (IPAddressUtil.isIPv6LiteralAddress(host)) {
		byte[] ipV6Address = IPAddressUtil.textToNumericFormatV6(host);
		host = getIPv6UrlRepresentation(ipV6Address);
	} else if (!IPAddressUtil.isIPv4LiteralAddress(host)) {
		try {
			// We don't allow these in hostnames
			Preconditions.checkArgument(!host.startsWith("."));
			Preconditions.checkArgument(!host.endsWith("."));
			Preconditions.checkArgument(!host.contains(":"));
		} catch (Exception e) {
			throw new IllegalConfigurationException("The configured hostname is not valid", e);
		}
	}

	return host;
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RocksDBStateBackendFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RocksDBStateBackend createFromConfig(Configuration config, ClassLoader classLoader)
		throws IllegalConfigurationException, IOException {

	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDirURI = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDirURI == null) {
		throw new IllegalConfigurationException(
			"Cannot create the RocksDB state backend: The configuration does not specify the " +
			"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	return new RocksDBStateBackend(checkpointDirURI).configure(config, classLoader);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: YarnClusterDescriptorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
		flinkConfiguration,
		yarnConfiguration,
		temporaryFolder.getRoot().getAbsolutePath(),
		yarnClient,
		true);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(Integer.MAX_VALUE)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}
 
Example 19
Source Project: Flink-CEPplus   Source File: YarnClusterDescriptorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigOverwrite() throws ClusterDeploymentException {
	Configuration configuration = new Configuration();
	// overwrite vcores in config
	configuration.setInteger(YarnConfigOptions.VCORES, Integer.MAX_VALUE);
	configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
		configuration,
		yarnConfiguration,
		temporaryFolder.getRoot().getAbsolutePath(),
		yarnClient,
		true);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	// configure slots
	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(1)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: SecurityConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
private void validate() {
	if (!StringUtils.isBlank(keytab)) {
		// principal is required
		if (StringUtils.isBlank(principal)) {
			throw new IllegalConfigurationException("Kerberos login configuration is invalid; keytab requires a principal.");
		}

		// check the keytab is readable
		File keytabFile = new File(keytab);
		if (!keytabFile.exists() || !keytabFile.isFile() || !keytabFile.canRead()) {
			throw new IllegalConfigurationException("Kerberos login configuration is invalid; keytab is unreadable");
		}
	}
}
 
Example 21
Source Project: flink   Source File: SSLUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static String getAndCheckOption(Configuration config, ConfigOption<String> primaryOption, ConfigOption<String> fallbackOption) {
	String value = config.getString(primaryOption, config.getString(fallbackOption));
	if (value != null) {
		return value;
	}
	else {
		throw new IllegalConfigurationException("The config option " + primaryOption.key() +
				" or " + fallbackOption.key() + " is missing.");
	}
}
 
Example 22
Source Project: flink   Source File: InputProcessorUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static BufferStorage createBufferStorage(
		CheckpointingMode checkpointMode,
		IOManager ioManager,
		int pageSize,
		Configuration taskManagerConfig,
		String taskName) throws IOException {
	switch (checkpointMode) {
		case EXACTLY_ONCE: {
			long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
			if (!(maxAlign == -1 || maxAlign > 0)) {
				throw new IllegalConfigurationException(
					TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
						+ " must be positive or -1 (infinite)");
			}

			if (taskManagerConfig.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_CREDIT_MODEL)) {
				return new CachedBufferStorage(pageSize, maxAlign, taskName);
			} else {
				return new BufferSpiller(ioManager, pageSize, maxAlign, taskName);
			}
		}
		case AT_LEAST_ONCE:
			return new EmptyBufferStorage();
		default:
			throw new UnsupportedOperationException("Unrecognized Checkpointing Mode: " + checkpointMode);
	}
}
 
Example 23
Source Project: flink   Source File: SSLUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that REST Client SSL creation fails with bad SSL configuration.
 */
@Test
public void testRESTClientSSLMissingPassword() throws Exception {
	Configuration config = new Configuration();
	config.setBoolean(SecurityOptions.SSL_REST_ENABLED, true);
	config.setString(SecurityOptions.SSL_REST_TRUSTSTORE, TRUST_STORE_PATH);

	try {
		SSLUtils.createRestClientSSLEngineFactory(config);
		fail("exception expected");
	} catch (IllegalConfigurationException ignored) {}
}
 
Example 24
Source Project: flink   Source File: SSLUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link SSLHandlerFactory} to be used by the REST Servers.
 *
 * @param config The application configuration.
 */
public static SSLHandlerFactory createRestServerSSLEngineFactory(final Configuration config) throws Exception {
	ClientAuth clientAuth = isRestSSLAuthenticationEnabled(config) ? ClientAuth.REQUIRE : ClientAuth.NONE;
	SslContext sslContext = createRestNettySSLContext(config, false, clientAuth);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for REST endpoints.");
	}

	return new SSLHandlerFactory(
			sslContext,
			-1,
			-1);
}
 
Example 25
Source Project: Flink-CEPplus   Source File: FailoverStrategyLoader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Loads a FailoverStrategy Factory from the given configuration.
 */
public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config, @Nullable Logger logger) {
	final String strategyParam = config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY);

	if (StringUtils.isNullOrWhitespaceOnly(strategyParam)) {
		if (logger != null) {
			logger.warn("Null config value for {} ; using default failover strategy (full restarts).",
					JobManagerOptions.EXECUTION_FAILOVER_STRATEGY.key());
		}

		return new RestartAllStrategy.Factory();
	}
	else {
		switch (strategyParam.toLowerCase()) {
			case FULL_RESTART_STRATEGY_NAME:
				return new RestartAllStrategy.Factory();

			case PIPELINED_REGION_RESTART_STRATEGY_NAME:
				return new RestartPipelinedRegionStrategy.Factory();

			case INDIVIDUAL_RESTART_STRATEGY_NAME:
				return new RestartIndividualStrategy.Factory();

			default:
				// we could interpret the parameter as a factory class name and instantiate that
				// for now we simply do not support this
				throw new IllegalConfigurationException("Unknown failover strategy: " + strategyParam);
		}
	}
}
 
Example 26
Source Project: Flink-CEPplus   Source File: ZooKeeperUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the configured ZooKeeper quorum (and removes whitespace, because ZooKeeper does not
 * tolerate it).
 */
public static String getZooKeeperEnsemble(Configuration flinkConf)
		throws IllegalConfigurationException {

	String zkQuorum = flinkConf.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new IllegalConfigurationException("No ZooKeeper quorum specified in config.");
	}

	// Remove all whitespace
	zkQuorum = zkQuorum.replaceAll("\\s+", "");

	return zkQuorum;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: ZooKeeperUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link FileSystemStateStorageHelper} instance.
 *
 * @param configuration {@link Configuration} object
 * @param prefix Prefix for the created files
 * @param <T> Type of the state objects
 * @return {@link FileSystemStateStorageHelper} instance
 * @throws IOException if file system state storage cannot be created
 */
public static <T extends Serializable> FileSystemStateStorageHelper<T> createFileSystemStateStorage(
		Configuration configuration,
		String prefix) throws IOException {

	String rootPath = configuration.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);

	if (rootPath == null || StringUtils.isBlank(rootPath)) {
		throw new IllegalConfigurationException("Missing high-availability storage path for metadata." +
				" Specify via configuration key '" + HighAvailabilityOptions.HA_STORAGE_PATH + "'.");
	} else {
		return new FileSystemStateStorageHelper<T>(rootPath, prefix);
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: ZooKeeperUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the configured {@link ZkClientACLMode}.
 *
 * @param config The config to parse
 * @return Configured ACL mode or the default defined by {@link HighAvailabilityOptions#ZOOKEEPER_CLIENT_ACL} if not
 * configured.
 */
public static ZkClientACLMode fromConfig(Configuration config) {
	String aclMode = config.getString(HighAvailabilityOptions.ZOOKEEPER_CLIENT_ACL);
	if (aclMode == null || aclMode.equalsIgnoreCase(ZkClientACLMode.OPEN.name())) {
		return ZkClientACLMode.OPEN;
	} else if (aclMode.equalsIgnoreCase(ZkClientACLMode.CREATOR.name())) {
		return ZkClientACLMode.CREATOR;
	} else {
		String message = "Unsupported ACL option: [" + aclMode + "] provided";
		LOG.error(message);
		throw new IllegalConfigurationException(message);
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a factory for SSL Server Sockets from the given configuration.
 * SSL Server Sockets are always part of internal communication.
 */
public static ServerSocketFactory createSSLServerSocketFactory(Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled");
	}

	String[] protocols = getEnabledProtocols(config);
	String[] cipherSuites = getEnabledCipherSuites(config);

	SSLServerSocketFactory factory = sslContext.getServerSocketFactory();
	return new ConfiguringSSLServerSocketFactory(factory, protocols, cipherSuites);
}
 
Example 30
Source Project: Flink-CEPplus   Source File: SSLUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a factory for SSL Client Sockets from the given configuration.
 * SSL Client Sockets are always part of internal communication.
 */
public static SocketFactory createSSLClientSocketFactory(Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled");
	}

	return sslContext.getSocketFactory();
}