org.apache.flink.configuration.RestOptions Java Examples
The following examples show how to use
org.apache.flink.configuration.RestOptions.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestServerEndpointConfigurationTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testBasicMapping() throws ConfigurationException { Configuration originalConfig = new Configuration(); originalConfig.setString(RestOptions.ADDRESS, ADDRESS); originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS); originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT); originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH); originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath()); final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig); Assert.assertEquals(ADDRESS, result.getRestAddress()); Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress()); Assert.assertEquals(BIND_PORT, result.getRestBindPortRange()); Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength()); Assert.assertThat( result.getUploadDir().toAbsolutePath().toString(), containsString(temporaryFolder.getRoot().getAbsolutePath())); }
Example #2
Source File: RestServerEndpointConfigurationTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testBasicMapping() throws ConfigurationException { Configuration originalConfig = new Configuration(); originalConfig.setString(RestOptions.ADDRESS, ADDRESS); originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS); originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT); originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH); originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath()); final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig); Assert.assertEquals(ADDRESS, result.getRestAddress()); Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress()); Assert.assertEquals(BIND_PORT, result.getRestBindPortRange()); Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength()); Assert.assertThat( result.getUploadDir().toAbsolutePath().toString(), containsString(temporaryFolder.getRoot().getAbsolutePath())); }
Example #3
Source File: DispatcherProcess.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Entrypoint of the DispatcherProcessEntryPoint. * * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher, * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum * "xyz:123:456"</code>. */ public static void main(String[] args) { try { ParameterTool params = ParameterTool.fromArgs(args); Configuration config = params.getConfiguration(); LOG.info("Configuration: {}.", config); config.setInteger(JobManagerOptions.PORT, 0); config.setString(RestOptions.BIND_PORT, "0"); final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); } catch (Throwable t) { LOG.error("Failed to start Dispatcher process", t); System.exit(1); } }
Example #4
Source File: TestingMiniClusterConfiguration.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public TestingMiniClusterConfiguration build() { final Configuration modifiedConfiguration = new Configuration(configuration); modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager); modifiedConfiguration.setString( RestOptions.ADDRESS, modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost")); modifiedConfiguration.setInteger( RestOptions.PORT, modifiedConfiguration.getInteger(RestOptions.PORT, 0)); return new TestingMiniClusterConfiguration( modifiedConfiguration, numTaskManagers, rpcServiceSharing, commonBindAddress, numberDispatcherResourceManagerComponents, localCommunication); }
Example #5
Source File: RestClientConfiguration.java From flink with Apache License 2.0 | 6 votes |
/** * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}. * * @param config configuration from which the REST client endpoint configuration should be created from * @return REST client endpoint configuration * @throws ConfigurationException if SSL was configured incorrectly */ public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException { Preconditions.checkNotNull(config); final SSLHandlerFactory sslHandlerFactory; if (SSLUtils.isRestSSLEnabled(config)) { try { sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config); } catch (Exception e) { throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e); } } else { sslHandlerFactory = null; } final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT); final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT); int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH); return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength); }
Example #6
Source File: FlinkHttpObjectAggregator.java From flink with Apache License 2.0 | 6 votes |
@Override protected void decode( final ChannelHandlerContext ctx, final HttpObject msg, final List<Object> out) throws Exception { try { super.decode(ctx, msg, out); } catch (final TooLongFrameException e) { HandlerUtils.sendErrorResponse( ctx, false, new ErrorResponseBody(String.format( e.getMessage() + " Try to raise [%s]", RestOptions.SERVER_MAX_CONTENT_LENGTH.key())), HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, responseHeaders); } }
Example #7
Source File: RestClientTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testConnectionTimeout() throws Exception { final Configuration config = new Configuration(); config.setLong(RestOptions.CONNECTION_TIMEOUT, 1); try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) { restClient.sendRequest( unroutableIp, 80, new TestMessageHeaders(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()) .get(60, TimeUnit.SECONDS); } catch (final ExecutionException e) { final Throwable throwable = ExceptionUtils.stripExecutionException(e); assertThat(throwable, instanceOf(ConnectTimeoutException.class)); assertThat(throwable.getMessage(), containsString(unroutableIp)); } }
Example #8
Source File: ClusterEntrypoint.java From flink with Apache License 2.0 | 6 votes |
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) { final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties()); final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties); final int restPort = entrypointClusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } final String hostname = entrypointClusterConfiguration.getHostname(); if (hostname != null) { configuration.setString(JobManagerOptions.ADDRESS, hostname); } return configuration; }
Example #9
Source File: RestClientConfiguration.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}. * * @param config configuration from which the REST client endpoint configuration should be created from * @return REST client endpoint configuration * @throws ConfigurationException if SSL was configured incorrectly */ public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException { Preconditions.checkNotNull(config); final SSLHandlerFactory sslHandlerFactory; if (SSLUtils.isRestSSLEnabled(config)) { try { sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config); } catch (Exception e) { throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e); } } else { sslHandlerFactory = null; } final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT); final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT); int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH); return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength); }
Example #10
Source File: TestingMiniClusterConfiguration.java From flink with Apache License 2.0 | 6 votes |
public TestingMiniClusterConfiguration build() { final Configuration modifiedConfiguration = new Configuration(configuration); modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager); modifiedConfiguration.setString( RestOptions.ADDRESS, modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost")); modifiedConfiguration.setInteger( RestOptions.PORT, modifiedConfiguration.getInteger(RestOptions.PORT, 0)); return new TestingMiniClusterConfiguration( modifiedConfiguration, numTaskManagers, rpcServiceSharing, commonBindAddress, numberDispatcherResourceManagerComponents, localCommunication); }
Example #11
Source File: LocalExecutor.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception { if (!configuration.contains(RestOptions.BIND_PORT)) { configuration.setString(RestOptions.BIND_PORT, "0"); } final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers( configuration.getInteger( ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER)) .setRpcServiceSharing(RpcServiceSharing.SHARED) .setNumSlotsPerTaskManager( configuration.getInteger( TaskManagerOptions.NUM_TASK_SLOTS, 1)) .build(); final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration); miniCluster.start(); configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); return miniCluster; }
Example #12
Source File: RestServerEndpointITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testRestServerBindPort() throws Exception { final int portRangeStart = 52300; final int portRangeEnd = 52400; final Configuration config = new Configuration(); config.setString(RestOptions.ADDRESS, "localhost"); config.setString(RestOptions.BIND_PORT, portRangeStart + "-" + portRangeEnd); final RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config); try (RestServerEndpoint serverEndpoint1 = new TestRestServerEndpoint(serverConfig, Collections.emptyList()); RestServerEndpoint serverEndpoint2 = new TestRestServerEndpoint(serverConfig, Collections.emptyList())) { serverEndpoint1.start(); serverEndpoint2.start(); assertNotEquals(serverEndpoint1.getServerAddress().getPort(), serverEndpoint2.getServerAddress().getPort()); assertThat(serverEndpoint1.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart))); assertThat(serverEndpoint1.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd))); assertThat(serverEndpoint2.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart))); assertThat(serverEndpoint2.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd))); } }
Example #13
Source File: RestServerEndpointITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testRestServerBindPort() throws Exception { final int portRangeStart = 52300; final int portRangeEnd = 52400; final Configuration config = new Configuration(); config.setString(RestOptions.ADDRESS, "localhost"); config.setString(RestOptions.BIND_PORT, portRangeStart + "-" + portRangeEnd); final RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config); try (RestServerEndpoint serverEndpoint1 = new TestRestServerEndpoint(serverConfig, Collections.emptyList()); RestServerEndpoint serverEndpoint2 = new TestRestServerEndpoint(serverConfig, Collections.emptyList())) { serverEndpoint1.start(); serverEndpoint2.start(); assertNotEquals(serverEndpoint1.getServerAddress().getPort(), serverEndpoint2.getServerAddress().getPort()); assertThat(serverEndpoint1.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart))); assertThat(serverEndpoint1.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd))); assertThat(serverEndpoint2.getServerAddress().getPort(), is(greaterThanOrEqualTo(portRangeStart))); assertThat(serverEndpoint2.getServerAddress().getPort(), is(lessThanOrEqualTo(portRangeEnd))); } }
Example #14
Source File: RestClientTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testConnectionTimeout() throws Exception { final Configuration config = new Configuration(); config.setLong(RestOptions.CONNECTION_TIMEOUT, 1); try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) { restClient.sendRequest( unroutableIp, 80, new TestMessageHeaders(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()) .get(60, TimeUnit.SECONDS); } catch (final ExecutionException e) { final Throwable throwable = ExceptionUtils.stripExecutionException(e); assertThat(throwable, instanceOf(ConnectTimeoutException.class)); assertThat(throwable.getMessage(), containsString(unroutableIp)); } }
Example #15
Source File: DispatcherProcess.java From flink with Apache License 2.0 | 6 votes |
/** * Entrypoint of the DispatcherProcessEntryPoint. * * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher, * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum * "xyz:123:456"</code>. */ public static void main(String[] args) { try { ParameterTool params = ParameterTool.fromArgs(args); Configuration config = params.getConfiguration(); LOG.info("Configuration: {}.", config); config.setInteger(JobManagerOptions.PORT, 0); config.setString(RestOptions.BIND_PORT, "0"); final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); } catch (Throwable t) { LOG.error("Failed to start Dispatcher process", t); System.exit(1); } }
Example #16
Source File: LocalExecutor.java From flink with Apache License 2.0 | 6 votes |
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception { if (!configuration.contains(RestOptions.BIND_PORT)) { configuration.setString(RestOptions.BIND_PORT, "0"); } final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers( configuration.getInteger( ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER)) .setRpcServiceSharing(RpcServiceSharing.SHARED) .setNumSlotsPerTaskManager( configuration.getInteger( TaskManagerOptions.NUM_TASK_SLOTS, 1)) .build(); final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration); miniCluster.start(); configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); return miniCluster; }
Example #17
Source File: JobRetrievalITCase.java From flink with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { final Configuration clientConfig = new Configuration(); clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0); clientConfig.setLong(RestOptions.RETRY_DELAY, 0); clientConfig.addAll(CLUSTER.getClientConfiguration()); client = new RestClusterClient<>( clientConfig, StandaloneClusterId.getInstance() ); }
Example #18
Source File: ExecutionEnvironment.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a {@link LocalEnvironment} for local program execution that also starts the * web monitoring UI. * * <p>The local execution environment will run the program in a multi-threaded fashion in * the same JVM as the environment was created in. It will use the parallelism specified in the * parameter. * * <p>If the configuration key 'rest.port' was set in the configuration, that particular * port will be used for the web UI. Otherwise, the default port (8081) will be used. */ @PublicEvolving public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) { checkNotNull(conf, "conf"); conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true); if (!conf.contains(RestOptions.PORT)) { // explicitly set this option so that it's not set to 0 later conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue()); } return createLocalEnvironment(conf, -1); }
Example #19
Source File: RemoteStreamExecutionEnvironmentTest.java From flink with Apache License 2.0 | 5 votes |
/** * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the cluster. */ @Test public void testPortForwarding() throws Exception { String host = "fakeHost"; int port = 99; JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) .thenReturn(expectedResult); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> { Object[] args = invocation.getArguments(); Configuration config = (Configuration) args[0]; Assert.assertEquals(host, config.getString(RestOptions.ADDRESS)); Assert.assertEquals(port, config.getInteger(RestOptions.PORT)); return mockedClient; } ); final Configuration clientConfiguration = new Configuration(); final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment( host, port, clientConfiguration); env.fromElements(1).map(x -> x * 2); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); }
Example #20
Source File: RestServerEndpointITCase.java From flink with Apache License 2.0 | 5 votes |
private static Configuration getBaseConfig() { final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress(); final Configuration config = new Configuration(); config.setString(RestOptions.BIND_PORT, "0"); config.setString(RestOptions.BIND_ADDRESS, loopbackAddress); config.setString(RestOptions.ADDRESS, loopbackAddress); config.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); config.setInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); return config; }
Example #21
Source File: MiniClusterResource.java From flink with Apache License 2.0 | 5 votes |
private void startMiniCluster() throws Exception { final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration()); configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath()); // we need to set this since a lot of test expect this because TestBaseUtils.startCluster() // enabled this by default if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) { configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); } if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) { configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE); } // set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters configuration.setInteger(JobManagerOptions.PORT, 0); configuration.setString(RestOptions.BIND_PORT, "0"); final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers()) .setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager()) .build(); miniCluster = new MiniCluster(miniClusterConfiguration); miniCluster.start(); final URI restAddress = miniCluster.getRestAddress().get(); createClientConfiguration(restAddress); }
Example #22
Source File: MultipartUploadResource.java From flink with Apache License 2.0 | 5 votes |
@Override public void before() throws Exception { temporaryFolder.create(); Configuration config = new Configuration(); config.setString(RestOptions.BIND_PORT, "0"); config.setString(RestOptions.ADDRESS, "localhost"); // set this to a lower value on purpose to test that files larger than the content limit are still accepted config.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, 1024 * 1024); configuredUploadDir = temporaryFolder.newFolder().toPath(); config.setString(WebOptions.UPLOAD_DIR, configuredUploadDir.toString()); RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config); RestfulGateway mockRestfulGateway = mock(RestfulGateway.class); final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway); file1 = temporaryFolder.newFile(); try (RandomAccessFile rw = new RandomAccessFile(file1, "rw")) { rw.setLength(1024 * 1024 * 64); } file2 = temporaryFolder.newFile(); Files.write(file2.toPath(), "world".getBytes(ConfigConstants.DEFAULT_CHARSET)); mixedHandler = new MultipartMixedHandler(mockGatewayRetriever, Arrays.asList(file1.toPath(), file2.toPath())); jsonHandler = new MultipartJsonHandler(mockGatewayRetriever); fileHandler = new MultipartFileHandler(mockGatewayRetriever, Arrays.asList(file1.toPath(), file2.toPath())); final List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = Arrays.asList( Tuple2.of(mixedHandler.getMessageHeaders(), mixedHandler), Tuple2.of(jsonHandler.getMessageHeaders(), jsonHandler), Tuple2.of(fileHandler.getMessageHeaders(), fileHandler)); serverEndpoint = new TestRestServerEndpoint(serverConfig, handlers); serverEndpoint.start(); serverAddress = serverEndpoint.getRestBaseUrl(); serverSocketAddress = serverEndpoint.getServerAddress(); }
Example #23
Source File: FileBufferReaderITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSequentialReading() throws Exception { // setup final Configuration configuration = new Configuration(); configuration.setString(RestOptions.BIND_PORT, "0"); configuration.setString(NettyShuffleEnvironmentOptions.NETWORK_BOUNDED_BLOCKING_SUBPARTITION_TYPE, "file"); final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers(parallelism) .setNumSlotsPerTaskManager(1) .build(); try (final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) { miniCluster.start(); final MiniClusterClient client = new MiniClusterClient(configuration, miniCluster); final JobGraph jobGraph = createJobGraph(); final CompletableFuture<JobSubmissionResult> submitFuture = client.submitJob(jobGraph); // wait for the submission to succeed final JobSubmissionResult result = submitFuture.get(); final CompletableFuture<JobResult> resultFuture = client.requestJobResult(result.getJobID()); final JobResult jobResult = resultFuture.get(); assertThat(jobResult.getSerializedThrowable().isPresent(), is(false)); } }
Example #24
Source File: SchedulingITCase.java From flink with Apache License 2.0 | 5 votes |
private void executeSchedulingTest(Configuration configuration) throws Exception { configuration.setString(RestOptions.BIND_PORT, "0"); final long slotIdleTimeout = 50L; configuration.setLong(JobManagerOptions.SLOT_IDLE_TIMEOUT, slotIdleTimeout); final int parallelism = 4; final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers(parallelism) .setNumSlotsPerTaskManager(1) .build(); try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) { miniCluster.start(); MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster); JobGraph jobGraph = createJobGraph(slotIdleTimeout << 1, parallelism); CompletableFuture<JobSubmissionResult> submissionFuture = miniClusterClient.submitJob(jobGraph); // wait for the submission to succeed JobSubmissionResult jobSubmissionResult = submissionFuture.get(); CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobSubmissionResult.getJobID()); JobResult jobResult = resultFuture.get(); assertThat(jobResult.getSerializedThrowable().isPresent(), is(false)); } }
Example #25
Source File: RestClusterClientTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests that command line options override the configuration settings. */ @Test public void testRESTManualConfigurationOverride() throws Exception { final String configuredHostname = "localhost"; final int configuredPort = 1234; final Configuration configuration = new Configuration(); configuration.setString(JobManagerOptions.ADDRESS, configuredHostname); configuration.setInteger(JobManagerOptions.PORT, configuredPort); configuration.setString(RestOptions.ADDRESS, configuredHostname); configuration.setInteger(RestOptions.PORT, configuredPort); final DefaultCLI defaultCLI = new DefaultCLI(configuration); final String manualHostname = "123.123.123.123"; final int manualPort = 4321; final String[] args = {"-m", manualHostname + ':' + manualPort}; CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false); final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(commandLine); final RestClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine)); URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get(); assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname)); assertThat(webMonitorBaseUrl.getPort(), equalTo(manualPort)); }
Example #26
Source File: ClusterClientFactory.java From alchemy with Apache License 2.0 | 5 votes |
public static FlinkClient createRestClient(StandaloneClusterInfo clusterInfo, JarLoader jarLoader) throws Exception { Configuration configuration = new Configuration(); if (clusterInfo.getProperties() != null) { setProperties(clusterInfo.getProperties(), configuration); } configuration.setString(JobManagerOptions.ADDRESS, clusterInfo.getAddress()); configuration.setInteger(JobManagerOptions.PORT, clusterInfo.getPort()); configuration.setInteger(RestOptions.PORT, clusterInfo.getPort()); return createClient(configuration, jarLoader, clusterInfo.getDependencies(), clusterInfo.getWebInterfaceUrl()); }
Example #27
Source File: JobManagerWatermarkTrackerTest.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setUp() throws Exception { final Configuration config = new Configuration(); config.setInteger(RestOptions.PORT, 0); final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(config) .setNumTaskManagers(1) .setNumSlotsPerTaskManager(1) .build(); flink = new MiniCluster(miniClusterConfiguration); flink.start(); }
Example #28
Source File: RemoteStreamExecutionEnvironmentTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the cluster. */ @Test public void testPortForwarding() throws Exception { String host = "fakeHost"; int port = 99; JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) .thenReturn(expectedResult); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> { Object[] args = invocation.getArguments(); Configuration config = (Configuration) args[0]; Assert.assertEquals(host, config.getString(RestOptions.ADDRESS)); Assert.assertEquals(port, config.getInteger(RestOptions.PORT)); return mockedClient; } ); final Configuration clientConfiguration = new Configuration(); final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment( host, port, clientConfiguration); env.fromElements(1).map(x -> x * 2); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); }
Example #29
Source File: StreamExecutionEnvironment.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a {@link LocalStreamEnvironment} for local program execution that also starts the * web monitoring UI. * * <p>The local execution environment will run the program in a multi-threaded fashion in * the same JVM as the environment was created in. It will use the parallelism specified in the * parameter. * * <p>If the configuration key 'rest.port' was set in the configuration, that particular * port will be used for the web UI. Otherwise, the default port (8081) will be used. */ @PublicEvolving public static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) { checkNotNull(conf, "conf"); conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true); if (!conf.contains(RestOptions.PORT)) { // explicitly set this option so that it's not set to 0 later conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue()); } return createLocalEnvironment(defaultLocalParallelism, conf); }
Example #30
Source File: JobManagerWatermarkTrackerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testUpateWatermark() throws Exception { final Configuration clientConfiguration = new Configuration(); clientConfiguration.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0); final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment( flink.getRestAddress().get().getHost(), flink.getRestAddress().get().getPort(), clientConfiguration); env.addSource(new TestSourceFunction(new JobManagerWatermarkTracker("fakeId"))) .addSink(new SinkFunction<Integer>() {}); env.execute(); }