org.apache.flink.runtime.net.SSLUtils Java Examples

The following examples show how to use org.apache.flink.runtime.net.SSLUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestClientConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example #2
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Get address of web monitor from configuration.
 *
 * @param configuration Configuration contains those for WebMonitor.
 * @param resolution Whether to try address resolution of the given hostname or not.
 *                   This allows to fail fast in case that the hostname cannot be resolved.
 * @return Address of WebMonitor.
 */
public static String getWebMonitorAddress(
	Configuration configuration,
	HighAvailabilityServicesUtils.AddressResolution resolution) throws UnknownHostException {
	final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS), "%s must be set", RestOptions.ADDRESS.key());

	if (resolution == HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION) {
		// Fail fast if the hostname cannot be resolved
		//noinspection ResultOfMethodCallIgnored
		InetAddress.getByName(address);
	}

	final int port = configuration.getInteger(RestOptions.PORT);
	final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
	final String protocol = enableSSL ? "https://" : "http://";

	return String.format("%s%s:%s", protocol, address, port);
}
 
Example #3
Source File: RestClientConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example #4
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #5
Source File: RestClientConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example #6
Source File: AkkaRpcServiceUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #7
Source File: AkkaRpcServiceUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param hostname The hostname or address where the target RPC service is listening.
 * @param port The port where the target RPC service is listening.
 * @param endpointName The name of the RPC endpoint.
 * @param addressResolution Whether to try address resolution of the given hostname or not.
 *                          This allows to fail fast in case that the hostname cannot be resolved.
 * @param config The configuration from which to deduce further settings.
 *
 * @return The RPC URL of the specified RPC endpoint.
 */
public static String getRpcUrl(
	String hostname,
	int port,
	String endpointName,
	HighAvailabilityServicesUtils.AddressResolution addressResolution,
	Configuration config) throws UnknownHostException {

	checkNotNull(config, "config is null");

	final boolean sslEnabled = config.getBoolean(AkkaOptions.SSL_ENABLED) &&
			SSLUtils.isInternalSSLEnabled(config);

	return getRpcUrl(
		hostname,
		port,
		endpointName,
		addressResolution,
		sslEnabled ? AkkaProtocol.SSL_TCP : AkkaProtocol.TCP);
}
 
Example #8
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example #9
Source File: RestServerEndpointITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws Exception {
	config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath());

	defaultSSLContext = SSLContext.getDefault();
	defaultSSLSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
	final SSLContext sslClientContext = SSLUtils.createRestSSLContext(config, true);
	if (sslClientContext != null) {
		SSLContext.setDefault(sslClientContext);
		HttpsURLConnection.setDefaultSSLSocketFactory(sslClientContext.getSocketFactory());
	}

	RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);
	RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config);

	RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	testHandler = new TestHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionHandler testVersionHandler = new TestVersionHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler1 testVersionSelectionHandler1 = new TestVersionSelectionHandler1(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler2 testVersionSelectionHandler2 = new TestVersionSelectionHandler2(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	testUploadHandler = new TestUploadHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	final StaticFileServerHandler<RestfulGateway> staticFileServerHandler = new StaticFileServerHandler<>(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT,
		temporaryFolder.getRoot());

	final List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = Arrays.asList(
		Tuple2.of(new TestHeaders(), testHandler),
		Tuple2.of(TestUploadHeaders.INSTANCE, testUploadHandler),
		Tuple2.of(testVersionHandler.getMessageHeaders(), testVersionHandler),
		Tuple2.of(testVersionSelectionHandler1.getMessageHeaders(), testVersionSelectionHandler1),
		Tuple2.of(testVersionSelectionHandler2.getMessageHeaders(), testVersionSelectionHandler2),
		Tuple2.of(WebContentHandlerSpecification.getInstance(), staticFileServerHandler));

	serverEndpoint = new TestRestServerEndpoint(serverConfig, handlers);
	restClient = new TestRestClient(clientConfig);

	serverEndpoint.start();
	serverAddress = serverEndpoint.getServerAddress();
}
 
Example #10
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public boolean getSSLEnabled() {
	return config.getBoolean(NettyShuffleEnvironmentOptions.DATA_SSL_ENABLED)
		&& SSLUtils.isInternalSSLEnabled(config);
}
 
Example #11
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createServerSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalServerSSLEngineFactory(config) :
			null;
}
 
Example #12
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createClientSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalClientSSLEngineFactory(config) :
			null;
}
 
Example #13
Source File: BlobServer.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	final String bindHost = config.getOptional(JobManagerOptions.BIND_HOST).orElseGet(NetUtils::getWildcardIPAddress);

	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog, InetAddress.getByName(bindHost)));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 
Example #14
Source File: RestServerEndpointConfiguration.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST server endpoint configuration should be created from
 * @return REST server endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */
public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.ADDRESS),
		"%s must be set",
		RestOptions.ADDRESS.key());

	final String restBindAddress = config.getString(RestOptions.BIND_ADDRESS);
	final String portRangeDefinition = config.getString(RestOptions.BIND_PORT);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestServerSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLEngineFactory for REST server endpoint.", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final Path uploadDir = Paths.get(
		config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
		"flink-web-upload");

	final int maxContentLength = config.getInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH);

	final Map<String, String> responseHeaders = Collections.singletonMap(
		HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN,
		config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN));

	return new RestServerEndpointConfiguration(
		restAddress,
		restBindAddress,
		portRangeDefinition,
		sslHandlerFactory,
		uploadDir,
		maxContentLength,
		responseHeaders);
}
 
Example #15
Source File: RestServerEndpointITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws Exception {
	config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath());

	defaultSSLContext = SSLContext.getDefault();
	defaultSSLSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
	final SSLContext sslClientContext = SSLUtils.createRestSSLContext(config, true);
	if (sslClientContext != null) {
		SSLContext.setDefault(sslClientContext);
		HttpsURLConnection.setDefaultSSLSocketFactory(sslClientContext.getSocketFactory());
	}

	RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);
	RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config);

	RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	testHandler = new TestHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionHandler testVersionHandler = new TestVersionHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler1 testVersionSelectionHandler1 = new TestVersionSelectionHandler1(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler2 testVersionSelectionHandler2 = new TestVersionSelectionHandler2(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	testUploadHandler = new TestUploadHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	final StaticFileServerHandler<RestfulGateway> staticFileServerHandler = new StaticFileServerHandler<>(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT,
		temporaryFolder.getRoot());

	final List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = Arrays.asList(
		Tuple2.of(new TestHeaders(), testHandler),
		Tuple2.of(TestUploadHeaders.INSTANCE, testUploadHandler),
		Tuple2.of(testVersionHandler.getMessageHeaders(), testVersionHandler),
		Tuple2.of(testVersionSelectionHandler1.getMessageHeaders(), testVersionSelectionHandler1),
		Tuple2.of(testVersionSelectionHandler2.getMessageHeaders(), testVersionSelectionHandler2),
		Tuple2.of(WebContentHandlerSpecification.getInstance(), staticFileServerHandler));

	serverEndpoint = new TestRestServerEndpoint(serverConfig, handlers);
	restClient = new TestRestClient(clientConfig);

	serverEndpoint.start();
	serverAddress = serverEndpoint.getServerAddress();
}
 
Example #16
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public boolean getSSLEnabled() {
	return config.getBoolean(NettyShuffleEnvironmentOptions.DATA_SSL_ENABLED)
		&& SSLUtils.isInternalSSLEnabled(config);
}
 
Example #17
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createServerSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalServerSSLEngineFactory(config) :
			null;
}
 
Example #18
Source File: NettyConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createClientSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalClientSSLEngineFactory(config) :
			null;
}
 
Example #19
Source File: BlobServer.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 
Example #20
Source File: RestServerEndpointConfiguration.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST server endpoint configuration should be created from
 * @return REST server endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */
public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.ADDRESS),
		"%s must be set",
		RestOptions.ADDRESS.key());

	final String restBindAddress = config.getString(RestOptions.BIND_ADDRESS);
	final String portRangeDefinition = config.getString(RestOptions.BIND_PORT);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestServerSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLEngineFactory for REST server endpoint.", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final Path uploadDir = Paths.get(
		config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
		"flink-web-upload");

	final int maxContentLength = config.getInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH);

	final Map<String, String> responseHeaders = Collections.singletonMap(
		HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN,
		config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN));

	return new RestServerEndpointConfiguration(
		restAddress,
		restBindAddress,
		portRangeDefinition,
		sslHandlerFactory,
		uploadDir,
		maxContentLength,
		responseHeaders);
}
 
Example #21
Source File: RestServerEndpointITCase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws Exception {
	config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath());

	defaultSSLContext = SSLContext.getDefault();
	defaultSSLSocketFactory = HttpsURLConnection.getDefaultSSLSocketFactory();
	final SSLContext sslClientContext = SSLUtils.createRestClientSSLContext(config);
	if (sslClientContext != null) {
		SSLContext.setDefault(sslClientContext);
		HttpsURLConnection.setDefaultSSLSocketFactory(sslClientContext.getSocketFactory());
	}

	RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);
	RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config);

	RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	testHandler = new TestHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionHandler testVersionHandler = new TestVersionHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler1 testVersionSelectionHandler1 = new TestVersionSelectionHandler1(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	TestVersionSelectionHandler2 testVersionSelectionHandler2 = new TestVersionSelectionHandler2(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	testUploadHandler = new TestUploadHandler(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT);

	final StaticFileServerHandler<RestfulGateway> staticFileServerHandler = new StaticFileServerHandler<>(
		mockGatewayRetriever,
		RpcUtils.INF_TIMEOUT,
		temporaryFolder.getRoot());

	final List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = Arrays.asList(
		Tuple2.of(new TestHeaders(), testHandler),
		Tuple2.of(TestUploadHeaders.INSTANCE, testUploadHandler),
		Tuple2.of(testVersionHandler.getMessageHeaders(), testVersionHandler),
		Tuple2.of(testVersionSelectionHandler1.getMessageHeaders(), testVersionSelectionHandler1),
		Tuple2.of(testVersionSelectionHandler2.getMessageHeaders(), testVersionSelectionHandler2),
		Tuple2.of(WebContentHandlerSpecification.getInstance(), staticFileServerHandler));

	serverEndpoint = new TestRestServerEndpoint(serverConfig, handlers);
	restClient = new TestRestClient(clientConfig);

	serverEndpoint.start();
	serverAddress = serverEndpoint.getServerAddress();
}
 
Example #22
Source File: NettyConfig.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public boolean getSSLEnabled() {
	return config.getBoolean(TaskManagerOptions.DATA_SSL_ENABLED)
		&& SSLUtils.isInternalSSLEnabled(config);
}
 
Example #23
Source File: NettyConfig.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createServerSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalServerSSLEngineFactory(config) :
			null;
}
 
Example #24
Source File: NettyConfig.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Nullable
public SSLHandlerFactory createClientSSLEngineFactory() throws Exception {
	return getSSLEnabled() ?
			SSLUtils.createInternalClientSSLEngineFactory(config) :
			null;
}
 
Example #25
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example #26
Source File: BlobServer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 
Example #27
Source File: RestServerEndpointConfiguration.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST server endpoint configuration should be created from
 * @return REST server endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */
public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.ADDRESS),
		"%s must be set",
		RestOptions.ADDRESS.key());

	final String restBindAddress = config.getString(RestOptions.BIND_ADDRESS);
	final String portRangeDefinition = config.getString(RestOptions.BIND_PORT);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestServerSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLEngineFactory for REST server endpoint.", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final Path uploadDir = Paths.get(
		config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
		"flink-web-upload");

	final int maxContentLength = config.getInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH);

	final Map<String, String> responseHeaders = Collections.singletonMap(
		HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN,
		config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN));

	return new RestServerEndpointConfiguration(
		restAddress,
		restBindAddress,
		portRangeDefinition,
		sslHandlerFactory,
		uploadDir,
		maxContentLength,
		responseHeaders);
}