org.apache.flink.client.deployment.StandaloneClusterId Java Examples

@Test
public void testTriggerSavepointFailureIllegalJobID() throws Exception {
	replaceStdOutAndStdErr();

	try {
		CliFrontend frontend = new MockedCliFrontend(new RestClusterClient<>(getConfiguration(), StandaloneClusterId.getInstance()));

		String[] parameters = { "invalid job id" };
		try {
			frontend.savepoint(parameters);
			fail("Should have failed.");
		} catch (CliArgsException e) {
			assertThat(e.getMessage(), Matchers.containsString("Cannot parse JobID"));
		}
	}
	finally {
		restoreStdOutAndStdErr();
	}
}
 

@Test
public void testTriggerSavepointFailureIllegalJobID() throws Exception {
	replaceStdOutAndStdErr();

	try {
		CliFrontend frontend = new MockedCliFrontend(new RestClusterClient<>(getConfiguration(), StandaloneClusterId.getInstance()));

		String[] parameters = { "invalid job id" };
		try {
			frontend.savepoint(parameters);
			fail("Should have failed.");
		} catch (CliArgsException e) {
			assertThat(e.getMessage(), Matchers.containsString("Cannot parse JobID"));
		}
	}
	finally {
		restoreStdOutAndStdErr();
	}
}
 

@Test
public void testTriggerSavepointFailureIllegalJobID() throws Exception {
	replaceStdOutAndStdErr();

	try {
		CliFrontend frontend = new MockedCliFrontend(new RestClusterClient<>(getConfiguration(), StandaloneClusterId.getInstance()));

		String[] parameters = { "invalid job id" };
		try {
			frontend.savepoint(parameters);
			fail("Should have failed.");
		} catch (CliArgsException e) {
			assertThat(e.getMessage(), Matchers.containsString("Cannot parse JobID"));
		}
	}
	finally {
		restoreStdOutAndStdErr();
	}
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(REST_CONFIG);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0,
		null);
}
 

private ClusterClient<?> getClusterClient(AbstractCustomCommandLine defaultCLI, CommandLine commandLine) throws FlinkException {
	final ClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
	final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<StandaloneClusterId> clusterFactory = serviceLoader.getClusterClientFactory(executorConfig);
	checkState(clusterFactory != null);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = clusterFactory.createClusterDescriptor(executorConfig);
	return clusterDescriptor.retrieve(clusterFactory.getClusterId(executorConfig)).getClusterClient();
}
 

DisposeSavepointClusterClient(
	Function<String, CompletableFuture<Acknowledge>> disposeSavepointFunction,
	Configuration configuration) throws Exception {
	super(configuration, StandaloneClusterId.getInstance());

	this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction);
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testRESTManualConfigurationOverride() throws Exception {
	final String configuredHostname = "localhost";
	final int configuredPort = 1234;
	final Configuration configuration = new Configuration();

	configuration.setString(JobManagerOptions.ADDRESS, configuredHostname);
	configuration.setInteger(JobManagerOptions.PORT, configuredPort);
	configuration.setString(RestOptions.ADDRESS, configuredHostname);
	configuration.setInteger(RestOptions.PORT, configuredPort);

	final DefaultCLI defaultCLI = new DefaultCLI(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
	final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);

	final ClusterClientFactory<StandaloneClusterId> clusterFactory = serviceLoader.getClusterClientFactory(executorConfig);
	checkState(clusterFactory != null);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = clusterFactory.createClusterDescriptor(executorConfig);
	final RestClusterClient<?> clusterClient = (RestClusterClient<?>) clusterDescriptor
			.retrieve(clusterFactory.getClusterId(executorConfig))
			.getClusterClient();

	URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get();
	assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname));
	assertThat(webMonitorBaseUrl.getPort(), equalTo(manualPort));
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(restConfig);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		createRestClient(),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0);
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(REST_CONFIG);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0);
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testManualConfigurationOverride() throws Exception {
	final String localhost = "localhost";
	final int port = 1234;
	final Configuration configuration = getConfiguration();

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(manualHostname));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(manualPort));
}
 

/**
 * Tests that the configuration is properly passed via the DefaultCLI to the
 * created ClusterDescriptor.
 */
@Test
public void testConfigurationPassing() throws Exception {
	final Configuration configuration = getConfiguration();

	final String localhost = "localhost";
	final int port = 1234;

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String[] args = {};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(localhost));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(port));
}
 

DisposeSavepointClusterClient(
	Function<String, CompletableFuture<Acknowledge>> disposeSavepointFunction,
	Configuration configuration) throws Exception {
	super(configuration, StandaloneClusterId.getInstance());

	this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction);
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(restConfig);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		createRestClient(),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0,
		null);
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testManualConfigurationOverride() throws Exception {
	final String localhost = "localhost";
	final int port = 1234;
	final Configuration configuration = getConfiguration();

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(manualHostname));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(manualPort));
}
 

/**
 * Tests that the configuration is properly passed via the DefaultCLI to the
 * created ClusterDescriptor.
 */
@Test
public void testConfigurationPassing() throws Exception {
	final Configuration configuration = getConfiguration();

	final String localhost = "localhost";
	final int port = 1234;

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String[] args = {};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(localhost));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(port));
}
 

DisposeSavepointClusterClient(
	Function<String, CompletableFuture<Acknowledge>> disposeSavepointFunction,
	Configuration configuration) throws Exception {
	super(configuration, StandaloneClusterId.getInstance());

	this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction);
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(restConfig);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		createRestClient(),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0,
		null);
}
 

private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception {
	final Configuration clientConfig = new Configuration(REST_CONFIG);
	clientConfig.setInteger(RestOptions.PORT, port);
	return new RestClusterClient<>(
		clientConfig,
		new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor),
		StandaloneClusterId.getInstance(),
		(attempt) -> 0,
		null);
}
 

@Override
@Nullable
public StandaloneClusterId getClusterId(CommandLine commandLine) {
	return StandaloneClusterId.getInstance();
}
 

/**
 * Use a map function that references a 100MB byte array.
 */
@Test
public void bigDataInMap() throws Exception {

	final byte[] data = new byte[16 * 1024 * 1024]; // 16 MB
	rnd.nextBytes(data); // use random data so that Java does not optimise it away
	data[1] = 0;
	data[3] = 0;
	data[5] = 0;

	CollectingSink resultSink = new CollectingSink();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataStream<Integer> src = env.fromElements(1, 3, 5);

	src.map(new MapFunction<Integer, String>() {
		private static final long serialVersionUID = 1L;

		@Override
		public String map(Integer value) throws Exception {
			return "x " + value + " " + data[value];
		}
	}).addSink(resultSink);

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	final RestClusterClient<StandaloneClusterId> restClusterClient = new RestClusterClient<>(
		MINI_CLUSTER_RESOURCE.getClientConfiguration(),
		StandaloneClusterId.getInstance());

	try {
		restClusterClient.setDetached(false);
		restClusterClient.submitJob(jobGraph, BigUserProgramJobSubmitITCase.class.getClassLoader());

		List<String> expected = Arrays.asList("x 1 0", "x 3 0", "x 5 0");

		List<String> result = CollectingSink.result;

		Collections.sort(expected);
		Collections.sort(result);

		assertEquals(expected, result);
	} finally {
		restClusterClient.shutdown();
	}
}
 

/**
 * Use a map function that references a 100MB byte array.
 */
@Test
public void bigDataInMap() throws Exception {

	final byte[] data = new byte[16 * 1024 * 1024]; // 16 MB
	rnd.nextBytes(data); // use random data so that Java does not optimise it away
	data[1] = 0;
	data[3] = 0;
	data[5] = 0;

	CollectingSink resultSink = new CollectingSink();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataStream<Integer> src = env.fromElements(1, 3, 5);

	src.map(new MapFunction<Integer, String>() {
		private static final long serialVersionUID = 1L;

		@Override
		public String map(Integer value) throws Exception {
			return "x " + value + " " + data[value];
		}
	}).addSink(resultSink);

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	final RestClusterClient<StandaloneClusterId> restClusterClient = new RestClusterClient<>(
		MINI_CLUSTER_RESOURCE.getClientConfiguration(),
		StandaloneClusterId.getInstance());

	try {
		ClientUtils.submitJobAndWaitForResult(restClusterClient, jobGraph, BigUserProgramJobSubmitITCase.class.getClassLoader());

		List<String> expected = Arrays.asList("x 1 0", "x 3 0", "x 5 0");

		List<String> result = CollectingSink.result;

		Collections.sort(expected);
		Collections.sort(result);

		assertEquals(expected, result);
	} finally {
		restClusterClient.close();
	}
}
 

TestingClusterClient(CompletableFuture<Tuple2<JobID, Integer>> rescaleJobFuture, Configuration configuration) throws Exception {
	super(configuration, StandaloneClusterId.getInstance());

	this.rescaleJobFuture = rescaleJobFuture;
}
 

@Override
@Nullable
public StandaloneClusterId getClusterId(CommandLine commandLine) {
	return StandaloneClusterId.getInstance();
}
 

/**
 * Use a map function that references a 100MB byte array.
 */
@Test
public void bigDataInMap() throws Exception {

	final byte[] data = new byte[16 * 1024 * 1024]; // 16 MB
	rnd.nextBytes(data); // use random data so that Java does not optimise it away
	data[1] = 0;
	data[3] = 0;
	data[5] = 0;

	CollectingSink resultSink = new CollectingSink();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataStream<Integer> src = env.fromElements(1, 3, 5);

	src.map(new MapFunction<Integer, String>() {
		private static final long serialVersionUID = 1L;

		@Override
		public String map(Integer value) throws Exception {
			return "x " + value + " " + data[value];
		}
	}).addSink(resultSink);

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	final RestClusterClient<StandaloneClusterId> restClusterClient = new RestClusterClient<>(
		MINI_CLUSTER_RESOURCE.getClientConfiguration(),
		StandaloneClusterId.getInstance());

	try {
		restClusterClient.setDetached(false);
		restClusterClient.submitJob(jobGraph, BigUserProgramJobSubmitITCase.class.getClassLoader());

		List<String> expected = Arrays.asList("x 1 0", "x 3 0", "x 5 0");

		List<String> result = CollectingSink.result;

		Collections.sort(expected);
		Collections.sort(result);

		assertEquals(expected, result);
	} finally {
		restClusterClient.shutdown();
	}
}