org.apache.flink.configuration.RestOptions Java Examples

The following examples show how to use org.apache.flink.configuration.RestOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestServerEndpointConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasicMapping() throws ConfigurationException {
	Configuration originalConfig = new Configuration();
	originalConfig.setString(RestOptions.ADDRESS, ADDRESS);
	originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS);
	originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT);
	originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH);
	originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath());

	final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig);
	Assert.assertEquals(ADDRESS, result.getRestAddress());
	Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress());
	Assert.assertEquals(BIND_PORT, result.getRestBindPortRange());
	Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength());
	Assert.assertThat(
		result.getUploadDir().toAbsolutePath().toString(),
		containsString(temporaryFolder.getRoot().getAbsolutePath()));
}
 
Example #2
Source File: RestServerEndpointConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasicMapping() throws ConfigurationException {
	Configuration originalConfig = new Configuration();
	originalConfig.setString(RestOptions.ADDRESS, ADDRESS);
	originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS);
	originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT);
	originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH);
	originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath());

	final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig);
	Assert.assertEquals(ADDRESS, result.getRestAddress());
	Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress());
	Assert.assertEquals(BIND_PORT, result.getRestBindPortRange());
	Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength());
	Assert.assertThat(
		result.getUploadDir().toAbsolutePath().toString(),
		containsString(temporaryFolder.getRoot().getAbsolutePath()));
}
 
Example #3
Source File: DispatcherProcess.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example #4
Source File: TestingMiniClusterConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TestingMiniClusterConfiguration build() {
	final Configuration modifiedConfiguration = new Configuration(configuration);
	modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager);
	modifiedConfiguration.setString(
		RestOptions.ADDRESS,
		modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost"));
	modifiedConfiguration.setInteger(
		RestOptions.PORT,
		modifiedConfiguration.getInteger(RestOptions.PORT, 0));

	return new TestingMiniClusterConfiguration(
		modifiedConfiguration,
		numTaskManagers,
		rpcServiceSharing,
		commonBindAddress,
		numberDispatcherResourceManagerComponents,
		localCommunication);
}
 
Example #5
Source File: RestClientConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example #6
Source File: FlinkHttpObjectAggregator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected void decode(
		final ChannelHandlerContext ctx,
		final HttpObject msg,
		final List<Object> out) throws Exception {

	try {
		super.decode(ctx, msg, out);
	} catch (final TooLongFrameException e) {
		HandlerUtils.sendErrorResponse(
			ctx,
			false,
			new ErrorResponseBody(String.format(
				e.getMessage() + " Try to raise [%s]",
				RestOptions.SERVER_MAX_CONTENT_LENGTH.key())),
			HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE,
			responseHeaders);
	}
}
 
Example #7
Source File: RestClientTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 
Example #8
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example #9
Source File: RestClientConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 
Example #10
Source File: TestingMiniClusterConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestingMiniClusterConfiguration build() {
	final Configuration modifiedConfiguration = new Configuration(configuration);
	modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager);
	modifiedConfiguration.setString(
		RestOptions.ADDRESS,
		modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost"));
	modifiedConfiguration.setInteger(
		RestOptions.PORT,
		modifiedConfiguration.getInteger(RestOptions.PORT, 0));

	return new TestingMiniClusterConfiguration(
		modifiedConfiguration,
		numTaskManagers,
		rpcServiceSharing,
		commonBindAddress,
		numberDispatcherResourceManagerComponents,
		localCommunication);
}
 
Example #11
Source File: LocalExecutor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 
Example #12
Source File: RestServerEndpointITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testRestServerBindPort() throws Exception {
	final int portRangeStart = 52300;
	final int portRangeEnd = 52400;
	final Configuration config = new Configuration();
	config.setString(RestOptions.ADDRESS, "localhost");
	config.setString(RestOptions.BIND_PORT, portRangeStart + "-" + portRangeEnd);

	final RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);

	try (RestServerEndpoint serverEndpoint1 = new TestRestServerEndpoint(serverConfig, Collections.emptyList());
		RestServerEndpoint serverEndpoint2 = new TestRestServerEndpoint(serverConfig, Collections.emptyList())) {

		serverEndpoint1.start();
		serverEndpoint2.start();

		assertNotEquals(serverEndpoint1.getServerAddress().getPort(), serverEndpoint2.getServerAddress().getPort());

		assertThat(serverEndpoint1.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart)));
		assertThat(serverEndpoint1.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd)));

		assertThat(serverEndpoint2.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart)));
		assertThat(serverEndpoint2.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd)));
	}
}
 
Example #13
Source File: RestServerEndpointITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testRestServerBindPort() throws Exception {
	final int portRangeStart = 52300;
	final int portRangeEnd = 52400;
	final Configuration config = new Configuration();
	config.setString(RestOptions.ADDRESS, "localhost");
	config.setString(RestOptions.BIND_PORT, portRangeStart + "-" + portRangeEnd);

	final RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);

	try (RestServerEndpoint serverEndpoint1 = new TestRestServerEndpoint(serverConfig, Collections.emptyList());
		RestServerEndpoint serverEndpoint2 = new TestRestServerEndpoint(serverConfig, Collections.emptyList())) {

		serverEndpoint1.start();
		serverEndpoint2.start();

		assertNotEquals(serverEndpoint1.getServerAddress().getPort(), serverEndpoint2.getServerAddress().getPort());

		assertThat(serverEndpoint1.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart)));
		assertThat(serverEndpoint1.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd)));

		assertThat(serverEndpoint2.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart)));
		assertThat(serverEndpoint2.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd)));
	}
}
 
Example #14
Source File: RestClientTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 
Example #15
Source File: DispatcherProcess.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example #16
Source File: LocalExecutor.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 
Example #17
Source File: JobRetrievalITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 
Example #18
Source File: ExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 
Example #19
Source File: RemoteStreamExecutionEnvironmentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the cluster.
 */
@Test
public void testPortForwarding() throws Exception {

	String host = "fakeHost";
	int port = 99;
	JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null);

	RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class);
	when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
		.thenReturn(expectedResult);

	PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> {
			Object[] args = invocation.getArguments();
			Configuration config = (Configuration) args[0];

			Assert.assertEquals(host, config.getString(RestOptions.ADDRESS));
			Assert.assertEquals(port, config.getInteger(RestOptions.PORT));
			return mockedClient;
		}
	);

	final Configuration clientConfiguration = new Configuration();
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		host, port, clientConfiguration);
	env.fromElements(1).map(x -> x * 2);
	JobExecutionResult actualResult = env.execute("fakeJobName");
	Assert.assertEquals(expectedResult, actualResult);
}
 
Example #20
Source File: RestServerEndpointITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Configuration getBaseConfig() {
	final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress();

	final Configuration config = new Configuration();
	config.setString(RestOptions.BIND_PORT, "0");
	config.setString(RestOptions.BIND_ADDRESS, loopbackAddress);
	config.setString(RestOptions.ADDRESS, loopbackAddress);
	config.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH);
	config.setInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH);
	return config;
}
 
Example #21
Source File: MiniClusterResource.java    From flink with Apache License 2.0 5 votes vote down vote up
private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
 
Example #22
Source File: MultipartUploadResource.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void before() throws Exception {
	temporaryFolder.create();
	Configuration config = new Configuration();
	config.setString(RestOptions.BIND_PORT, "0");
	config.setString(RestOptions.ADDRESS, "localhost");
	// set this to a lower value on purpose to test that files larger than the content limit are still accepted
	config.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, 1024 * 1024);
	configuredUploadDir = temporaryFolder.newFolder().toPath();
	config.setString(WebOptions.UPLOAD_DIR, configuredUploadDir.toString());

	RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);

	RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	file1 = temporaryFolder.newFile();
	try (RandomAccessFile rw = new RandomAccessFile(file1, "rw")) {
		rw.setLength(1024 * 1024 * 64);
	}
	file2 = temporaryFolder.newFile();
	Files.write(file2.toPath(), "world".getBytes(ConfigConstants.DEFAULT_CHARSET));

	mixedHandler = new MultipartMixedHandler(mockGatewayRetriever, Arrays.asList(file1.toPath(), file2.toPath()));
	jsonHandler = new MultipartJsonHandler(mockGatewayRetriever);
	fileHandler = new MultipartFileHandler(mockGatewayRetriever, Arrays.asList(file1.toPath(), file2.toPath()));

	final List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = Arrays.asList(
		Tuple2.of(mixedHandler.getMessageHeaders(), mixedHandler),
		Tuple2.of(jsonHandler.getMessageHeaders(), jsonHandler),
		Tuple2.of(fileHandler.getMessageHeaders(), fileHandler));

	serverEndpoint = new TestRestServerEndpoint(serverConfig, handlers);

	serverEndpoint.start();
	serverAddress = serverEndpoint.getRestBaseUrl();
	serverSocketAddress = serverEndpoint.getServerAddress();
}
 
Example #23
Source File: FileBufferReaderITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSequentialReading() throws Exception {
	// setup
	final Configuration configuration = new Configuration();
	configuration.setString(RestOptions.BIND_PORT, "0");
	configuration.setString(NettyShuffleEnvironmentOptions.NETWORK_BOUNDED_BLOCKING_SUBPARTITION_TYPE, "file");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(parallelism)
		.setNumSlotsPerTaskManager(1)
		.build();

	try (final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		final MiniClusterClient client = new MiniClusterClient(configuration, miniCluster);
		final JobGraph jobGraph = createJobGraph();
		final CompletableFuture<JobSubmissionResult> submitFuture = client.submitJob(jobGraph);
		// wait for the submission to succeed
		final JobSubmissionResult result = submitFuture.get();

		final CompletableFuture<JobResult> resultFuture = client.requestJobResult(result.getJobID());
		final JobResult jobResult = resultFuture.get();

		assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
	}
}
 
Example #24
Source File: SchedulingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void executeSchedulingTest(Configuration configuration) throws Exception {
	configuration.setString(RestOptions.BIND_PORT, "0");

	final long slotIdleTimeout = 50L;
	configuration.setLong(JobManagerOptions.SLOT_IDLE_TIMEOUT, slotIdleTimeout);

	final int parallelism = 4;
	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(parallelism)
		.setNumSlotsPerTaskManager(1)
		.build();

	try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);

		JobGraph jobGraph = createJobGraph(slotIdleTimeout << 1, parallelism);
		CompletableFuture<JobSubmissionResult> submissionFuture = miniClusterClient.submitJob(jobGraph);

		// wait for the submission to succeed
		JobSubmissionResult jobSubmissionResult = submissionFuture.get();

		CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobSubmissionResult.getJobID());

		JobResult jobResult = resultFuture.get();

		assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
	}
}
 
Example #25
Source File: RestClusterClientTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testRESTManualConfigurationOverride() throws Exception {
	final String configuredHostname = "localhost";
	final int configuredPort = 1234;
	final Configuration configuration = new Configuration();

	configuration.setString(JobManagerOptions.ADDRESS, configuredHostname);
	configuration.setInteger(JobManagerOptions.PORT, configuredPort);
	configuration.setString(RestOptions.ADDRESS, configuredHostname);
	configuration.setInteger(RestOptions.PORT, configuredPort);

	final DefaultCLI defaultCLI = new DefaultCLI(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(commandLine);

	final RestClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get();
	assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname));
	assertThat(webMonitorBaseUrl.getPort(), equalTo(manualPort));
}
 
Example #26
Source File: ClusterClientFactory.java    From alchemy with Apache License 2.0 5 votes vote down vote up
public static FlinkClient createRestClient(StandaloneClusterInfo clusterInfo, JarLoader jarLoader) throws Exception {
    Configuration configuration = new Configuration();
    if (clusterInfo.getProperties() != null) {
        setProperties(clusterInfo.getProperties(), configuration);
    }
    configuration.setString(JobManagerOptions.ADDRESS, clusterInfo.getAddress());
    configuration.setInteger(JobManagerOptions.PORT, clusterInfo.getPort());
    configuration.setInteger(RestOptions.PORT, clusterInfo.getPort());
    return createClient(configuration, jarLoader, clusterInfo.getDependencies(), clusterInfo.getWebInterfaceUrl());
}
 
Example #27
Source File: JobManagerWatermarkTrackerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
	final Configuration config = new Configuration();
	config.setInteger(RestOptions.PORT, 0);

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(config)
		.setNumTaskManagers(1)
		.setNumSlotsPerTaskManager(1)
		.build();

	flink = new MiniCluster(miniClusterConfiguration);

	flink.start();
}
 
Example #28
Source File: RemoteStreamExecutionEnvironmentTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the cluster.
 */
@Test
public void testPortForwarding() throws Exception {

	String host = "fakeHost";
	int port = 99;
	JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null);

	RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class);
	when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
		.thenReturn(expectedResult);

	PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> {
			Object[] args = invocation.getArguments();
			Configuration config = (Configuration) args[0];

			Assert.assertEquals(host, config.getString(RestOptions.ADDRESS));
			Assert.assertEquals(port, config.getInteger(RestOptions.PORT));
			return mockedClient;
		}
	);

	final Configuration clientConfiguration = new Configuration();
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		host, port, clientConfiguration);
	env.fromElements(1).map(x -> x * 2);
	JobExecutionResult actualResult = env.execute("fakeJobName");
	Assert.assertEquals(expectedResult, actualResult);
}
 
Example #29
Source File: StreamExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link LocalStreamEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(defaultLocalParallelism, conf);
}
 
Example #30
Source File: JobManagerWatermarkTrackerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testUpateWatermark() throws Exception {
	final Configuration clientConfiguration = new Configuration();
	clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
		flink.getRestAddress().get().getHost(),
		flink.getRestAddress().get().getPort(),
		clientConfiguration);

	env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId")))
		.addSink(new SinkFunction<Integer>() {});
	env.execute();
}