org.apache.flink.client.cli.CliFrontend Java Examples

public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable);
	}

	System.exit(retCode);
}
 

/**
 * Test per-job yarn cluster
 *
 * <p>This also tests the prefixed CliFrontend options for the YARN case
 * We also test if the requested parallelism of 2 is passed through.
 * The parallelism is requested at the YARN client (-ys).
 */
@Test
public void perJobYarnCluster() throws IOException {
	LOG.info("Starting perJobYarnCluster()");
	addTestAppender(CliFrontend.class, Level.INFO);
	File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
	runWithArgs(new String[]{"run", "-m", "yarn-cluster",
			"-yj", flinkUberjar.getAbsolutePath(),
			"-yt", flinkLibFolder.getAbsolutePath(),
			"-yt", flinkShadedHadoopDir.getAbsolutePath(),
			"-yn", "1",
			"-ys", "2", //test that the job is executed with a DOP of 2
			"-yjm", "768m",
			"-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
		/* test succeeded after this string */
		"Program execution finished",
		/* prohibited strings: (to verify the parallelism) */
		// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
		new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
		RunTypes.CLI_FRONTEND, 0, true);
	LOG.info("Finished perJobYarnCluster()");
}
 

public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e, LOG);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable, LOG);
	}

	System.exit(retCode);
}
 

/**
 * Test per-job yarn cluster with the parallelism set at the CliFrontend instead of the YARN client.
 */
@Test
public void perJobYarnClusterWithParallelism() throws IOException {
	LOG.info("Starting perJobYarnClusterWithParallelism()");
	// write log messages to stdout as well, so that the runWithArgs() method
	// is catching the log output
	addTestAppender(CliFrontend.class, Level.INFO);
	File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
	runWithArgs(new String[]{"run",
			"-p", "2", //test that the job is executed with a DOP of 2
			"-m", "yarn-cluster",
			"-yj", flinkUberjar.getAbsolutePath(),
			"-yt", flinkLibFolder.getAbsolutePath(),
			"-yt", flinkShadedHadoopDir.getAbsolutePath(),
			"-yn", "1",
			"-ys", "2",
			"-yjm", "768m",
			"-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
			/* test succeeded after this string */
		"Program execution finished",
		/* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
		new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
		RunTypes.CLI_FRONTEND, 0, true);
	LOG.info("Finished perJobYarnClusterWithParallelism()");
}
 

public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable);
	}

	System.exit(retCode);
}
 

/**
 * Test per-job yarn cluster
 *
 * <p>This also tests the prefixed CliFrontend options for the YARN case
 * We also test if the requested parallelism of 2 is passed through.
 * The parallelism is requested at the YARN client (-ys).
 */
@Test
public void perJobYarnCluster() throws Exception {
	runTest(() -> {
		LOG.info("Starting perJobYarnCluster()");
		addTestAppender(CliFrontend.class, Level.INFO);
		File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
		runWithArgs(new String[]{"run", "-m", "yarn-cluster",
				"-yj", flinkUberjar.getAbsolutePath(),
				"-yt", flinkLibFolder.getAbsolutePath(),
				"-yt", flinkShadedHadoopDir.getAbsolutePath(),
				"-yn", "1",
				"-ys", "2", //test that the job is executed with a DOP of 2
				"-yjm", "768m",
				"-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
			/* test succeeded after this string */
			"Program execution finished",
			/* prohibited strings: (to verify the parallelism) */
			// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
			new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
			RunTypes.CLI_FRONTEND, 0, true);
		LOG.info("Finished perJobYarnCluster()");
	});
}
 

@Nonnull
YarnClusterDescriptor createYarnClusterDescriptor(org.apache.flink.configuration.Configuration flinkConfiguration) {
	final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
		flinkConfiguration,
		YARN_CONFIGURATION,
		CliFrontend.getConfigurationDirectoryFromEnv(),
		yarnClient,
		true);
	yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.toURI()));
	yarnClusterDescriptor.addShipFiles(Collections.singletonList(flinkLibFolder));
	return yarnClusterDescriptor;
}
 

@Override
public void run() {
	try {
		int returnValue;
		switch (type) {
			case YARN_SESSION:
				yCli = new FlinkYarnSessionCli(
					configuration,
					configurationDirectory,
					"",
					"",
					true);
				returnValue = yCli.run(args);
				break;
			case CLI_FRONTEND:
				try {
					CliFrontend cli = new CliFrontend(
						configuration,
						CliFrontend.loadCustomCommandLines(configuration, configurationDirectory));
					returnValue = cli.parseParameters(args);
				} catch (Exception e) {
					throw new RuntimeException("Failed to execute the following args with CliFrontend: "
						+ Arrays.toString(args), e);
				}
				break;
			default:
				throw new RuntimeException("Unknown type " + type);
		}

		if (returnValue != this.expectedReturnValue) {
			Assert.fail("The YARN session returned with unexpected value=" + returnValue + " expected=" + expectedReturnValue);
		}
	} catch (Throwable t) {
		LOG.info("Runner stopped with exception", t);
		// save error.
		this.runnerError = t;
	}
}
 

private void validateExecutorCLIisPrioritised(Configuration configuration, String[] argsUnderTest) throws IOException, CliArgsException {
	final List<CustomCommandLine> customCommandLines = CliFrontend.loadCustomCommandLines(
			configuration,
			tmp.newFile().getAbsolutePath());

	final CliFrontend cli = new CliFrontend(configuration, customCommandLines);
	final CommandLine commandLine = cli.getCommandLine(
			CliFrontendParser.getRunCommandOptions(),
			argsUnderTest,
			true);

	final CustomCommandLine customCommandLine = cli.validateAndGetActiveCommandLine(commandLine);
	assertTrue(customCommandLine instanceof GenericCLI);
}
 

private void runSubmission(boolean isDetached, boolean isStreaming) throws Exception {
  PipelineOptions options = PipelineOptionsFactory.create();
  options.setTempLocation(TEMP_FOLDER.getRoot().getPath());
  String jarPath =
      Iterables.getFirst(
          PipelineResources.detectClassPathResourcesToStage(getClass().getClassLoader(), options),
          null);

  try {
    throwExceptionOnSystemExit();
    ImmutableList.Builder<String> argsBuilder = ImmutableList.builder();
    argsBuilder.add("run").add("-c").add(getClass().getName());
    if (isDetached) {
      argsBuilder.add("-d");
    }
    argsBuilder.add(jarPath);

    FlinkSubmissionTest.streaming = isStreaming;
    FlinkSubmissionTest.expectedNumberOfJobs++;
    // Run end-to-end test
    CliFrontend.main(argsBuilder.build().toArray(new String[0]));
  } catch (SystemExitException e) {
    // The CliFrontend exited and we can move on to check if the job has finished
  } finally {
    restoreDefaultSystemExitBehavior();
  }

  waitUntilJobIsCompleted();
}
 

/**
 * Test per-job yarn cluster with the parallelism set at the CliFrontend instead of the YARN client.
 */
@Test
public void perJobYarnClusterWithParallelism() throws Exception {
	runTest(() -> {
		LOG.info("Starting perJobYarnClusterWithParallelism()");
		// write log messages to stdout as well, so that the runWithArgs() method
		// is catching the log output
		addTestAppender(CliFrontend.class, Level.INFO);
		File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
		runWithArgs(new String[]{"run",
				"-p", "2", //test that the job is executed with a DOP of 2
				"-m", "yarn-cluster",
				"-yj", flinkUberjar.getAbsolutePath(),
				"-yt", flinkLibFolder.getAbsolutePath(),
				"-yt", flinkShadedHadoopDir.getAbsolutePath(),
				"-yn", "1",
				"-ys", "2",
				"-yjm", "768m",
				"-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
			/* test succeeded after this string */
			"Program execution finished",
			/* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
			new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
			RunTypes.CLI_FRONTEND, 0, true);
		LOG.info("Finished perJobYarnClusterWithParallelism()");
	});
}
 

/**
 * Test per-job yarn cluster and memory calculations for off-heap use (see FLINK-7400) with the
 * same job as {@link #perJobYarnCluster()}.
 *
 * <p>This ensures that with (any) pre-allocated off-heap memory by us, there is some off-heap
 * memory remaining for Flink's libraries. Creating task managers will thus fail if no off-heap
 * memory remains.
 */
@Test
public void perJobYarnClusterOffHeap() throws Exception {
	runTest(() -> {
		LOG.info("Starting perJobYarnCluster()");
		addTestAppender(CliFrontend.class, Level.INFO);
		File exampleJarLocation = getTestJarPath("BatchWordCount.jar");

		// set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
		final long taskManagerMemoryMB = 1024;
		//noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
		final long networkBuffersMB = NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(
			(taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
			new Configuration()) >> 20;
		final long offHeapMemory = taskManagerMemoryMB
			- ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
			// cutoff memory (will be added automatically)
			- networkBuffersMB // amount of memory used for network buffers
			- 100; // reserve something for the Java heap space

		runWithArgs(new String[]{"run", "-m", "yarn-cluster",
				"-yj", flinkUberjar.getAbsolutePath(),
				"-yt", flinkLibFolder.getAbsolutePath(),
				"-yt", flinkShadedHadoopDir.getAbsolutePath(),
				"-yn", "1",
				"-ys", "2", //test that the job is executed with a DOP of 2
				"-yjm", "768m",
				"-ytm", taskManagerMemoryMB + "m",
				"-yD", "taskmanager.memory.off-heap=true",
				"-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
				"-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
			/* test succeeded after this string */
			"Program execution finished",
			/* prohibited strings: (to verify the parallelism) */
			// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
			new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
			RunTypes.CLI_FRONTEND, 0, true);
		LOG.info("Finished perJobYarnCluster()");
	});
}
 

@Override
public void run() {
	try {
		int returnValue;
		switch (type) {
			case YARN_SESSION:
				yCli = new FlinkYarnSessionCli(
					configuration,
					configurationDirectory,
					"",
					"",
					true);
				returnValue = yCli.run(args);
				break;
			case CLI_FRONTEND:
				try {
					CliFrontend cli = new CliFrontend(
						configuration,
						CliFrontend.loadCustomCommandLines(configuration, configurationDirectory));
					returnValue = cli.parseParameters(args);
				} catch (Exception e) {
					throw new RuntimeException("Failed to execute the following args with CliFrontend: "
						+ Arrays.toString(args), e);
				}
				break;
			default:
				throw new RuntimeException("Unknown type " + type);
		}

		if (returnValue != this.expectedReturnValue) {
			Assert.fail("The YARN session returned with unexpected value=" + returnValue + " expected=" + expectedReturnValue);
		}
	} catch (Throwable t) {
		LOG.info("Runner stopped with exception", t);
		// save error.
		this.runnerError = t;
	}
}
 

@Nonnull
YarnClusterDescriptor createYarnClusterDescriptor(org.apache.flink.configuration.Configuration flinkConfiguration) {
	final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
		flinkConfiguration,
		YARN_CONFIGURATION,
		CliFrontend.getConfigurationDirectoryFromEnv(),
		yarnClient,
		true);
	yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.toURI()));
	yarnClusterDescriptor.addShipFiles(Collections.singletonList(flinkLibFolder));
	return yarnClusterDescriptor;
}
 

/**
 * Test per-job yarn cluster and memory calculations for off-heap use (see FLINK-7400) with the
 * same job as {@link #perJobYarnCluster()}.
 *
 * <p>This ensures that with (any) pre-allocated off-heap memory by us, there is some off-heap
 * memory remaining for Flink's libraries. Creating task managers will thus fail if no off-heap
 * memory remains.
 */
@Test
public void perJobYarnClusterOffHeap() throws IOException {
	LOG.info("Starting perJobYarnCluster()");
	addTestAppender(CliFrontend.class, Level.INFO);
	File exampleJarLocation = getTestJarPath("BatchWordCount.jar");

	// set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
	final long taskManagerMemoryMB = 1024;
	//noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
	final long networkBuffersMB = TaskManagerServices
		.calculateNetworkBufferMemory(
			(taskManagerMemoryMB -
				ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
			new Configuration()) >> 20;
	final long offHeapMemory = taskManagerMemoryMB
		- ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
		// cutoff memory (will be added automatically)
		- networkBuffersMB // amount of memory used for network buffers
		- 100; // reserve something for the Java heap space

	runWithArgs(new String[]{"run", "-m", "yarn-cluster",
			"-yj", flinkUberjar.getAbsolutePath(),
			"-yt", flinkLibFolder.getAbsolutePath(),
			"-yt", flinkShadedHadoopDir.getAbsolutePath(),
			"-yn", "1",
			"-ys", "2", //test that the job is executed with a DOP of 2
			"-yjm", "768m",
			"-ytm", taskManagerMemoryMB + "m",
			"-yD", "taskmanager.memory.off-heap=true",
			"-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
			"-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
		/* test succeeded after this string */
		"Program execution finished",
		/* prohibited strings: (to verify the parallelism) */
		// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
		new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
		RunTypes.CLI_FRONTEND, 0, true);
	LOG.info("Finished perJobYarnCluster()");
}
 

@Override
public void run() {
	try {
		int returnValue;
		switch (type) {
			case YARN_SESSION:
				yCli = new FlinkYarnSessionCli(
					configuration,
					configurationDirectory,
					"",
					"",
					true);
				returnValue = yCli.run(args);
				break;
			case CLI_FRONTEND:
				try {
					CliFrontend cli = new CliFrontend(
						configuration,
						CliFrontend.loadCustomCommandLines(configuration, configurationDirectory));
					returnValue = cli.parseParameters(args);
				} catch (Exception e) {
					throw new RuntimeException("Failed to execute the following args with CliFrontend: "
						+ Arrays.toString(args), e);
				}
				break;
			default:
				throw new RuntimeException("Unknown type " + type);
		}

		if (returnValue != this.expectedReturnValue) {
			Assert.fail("The YARN session returned with unexpected value=" + returnValue + " expected=" + expectedReturnValue);
		}
	} catch (Throwable t) {
		LOG.info("Runner stopped with exception", t);
		// save error.
		this.runnerError = t;
	}
}
 

/**
 * This method returns once the "startedAfterString" has been seen.
 */
protected Runner startWithArgs(String[] args, String startedAfterString, RunTypes type) throws IOException {
	LOG.info("Running with args {}", Arrays.toString(args));

	outContent = new ByteArrayOutputStream();
	errContent = new ByteArrayOutputStream();
	PipedOutputStream out = new PipedOutputStream();
	PipedInputStream in = new PipedInputStream(out);
	PrintStream stdinPrintStream = new PrintStream(out);

	System.setOut(new PrintStream(outContent));
	System.setErr(new PrintStream(errContent));
	System.setIn(in);

	final int startTimeoutSeconds = 60;

	Runner runner = new Runner(
		args,
		flinkConfiguration,
		CliFrontend.getConfigurationDirectoryFromEnv(),
		type,
		0,
		stdinPrintStream);
	runner.setName("Frontend (CLI/YARN Client) runner thread (startWithArgs()).");
	runner.start();

	for (int second = 0; second <  startTimeoutSeconds; second++) {
		sleep(1000);
		// check output for correct TaskManager startup.
		if (outContent.toString().contains(startedAfterString)
				|| errContent.toString().contains(startedAfterString)) {
			LOG.info("Found expected output in redirected streams");
			return runner;
		}
		// check if thread died
		if (!runner.isAlive()) {
			resetStreamsAndSendOutput();
			if (runner.getRunnerError() != null) {
				throw new RuntimeException("Runner failed with exception.", runner.getRunnerError());
			}
			Assert.fail("Runner thread died before the test was finished.");
		}
	}

	resetStreamsAndSendOutput();
	Assert.fail("During the timeout period of " + startTimeoutSeconds + " seconds the " +
			"expected string did not show up");
	return null;
}
 

/**
 * This method returns once the "startedAfterString" has been seen.
 */
protected Runner startWithArgs(String[] args, String startedAfterString, RunTypes type) throws IOException {
	LOG.info("Running with args {}", Arrays.toString(args));

	outContent = new ByteArrayOutputStream();
	errContent = new ByteArrayOutputStream();
	PipedOutputStream out = new PipedOutputStream();
	PipedInputStream in = new PipedInputStream(out);
	PrintStream stdinPrintStream = new PrintStream(out);

	System.setOut(new PrintStream(outContent));
	System.setErr(new PrintStream(errContent));
	System.setIn(in);

	final int startTimeoutSeconds = 60;

	Runner runner = new Runner(
		args,
		flinkConfiguration,
		CliFrontend.getConfigurationDirectoryFromEnv(),
		type,
		0,
		stdinPrintStream);
	runner.setName("Frontend (CLI/YARN Client) runner thread (startWithArgs()).");
	runner.start();

	for (int second = 0; second <  startTimeoutSeconds; second++) {
		sleep(1000);
		// check output for correct TaskManager startup.
		if (outContent.toString().contains(startedAfterString)
				|| errContent.toString().contains(startedAfterString)) {
			LOG.info("Found expected output in redirected streams");
			return runner;
		}
		// check if thread died
		if (!runner.isAlive()) {
			resetStreamsAndSendOutput();
			if (runner.getRunnerError() != null) {
				throw new RuntimeException("Runner failed with exception.", runner.getRunnerError());
			}
			Assert.fail("Runner thread died before the test was finished.");
		}
	}

	resetStreamsAndSendOutput();
	Assert.fail("During the timeout period of " + startTimeoutSeconds + " seconds the " +
			"expected string did not show up");
	return null;
}
 

/**
 * This method returns once the "startedAfterString" has been seen.
 */
protected Runner startWithArgs(String[] args, String startedAfterString, RunTypes type) throws IOException {
	LOG.info("Running with args {}", Arrays.toString(args));

	outContent = new ByteArrayOutputStream();
	errContent = new ByteArrayOutputStream();
	PipedOutputStream out = new PipedOutputStream();
	PipedInputStream in = new PipedInputStream(out);
	PrintStream stdinPrintStream = new PrintStream(out);

	System.setOut(new PrintStream(outContent));
	System.setErr(new PrintStream(errContent));
	System.setIn(in);

	final int startTimeoutSeconds = 60;

	Runner runner = new Runner(
		args,
		flinkConfiguration,
		CliFrontend.getConfigurationDirectoryFromEnv(),
		type,
		0,
		stdinPrintStream);
	runner.setName("Frontend (CLI/YARN Client) runner thread (startWithArgs()).");
	runner.start();

	for (int second = 0; second <  startTimeoutSeconds; second++) {
		sleep(1000);
		// check output for correct TaskManager startup.
		if (outContent.toString().contains(startedAfterString)
				|| errContent.toString().contains(startedAfterString)) {
			LOG.info("Found expected output in redirected streams");
			return runner;
		}
		// check if thread died
		if (!runner.isAlive()) {
			resetStreamsAndSendOutput();
			if (runner.getRunnerError() != null) {
				throw new RuntimeException("Runner failed with exception.", runner.getRunnerError());
			}
			Assert.fail("Runner thread died before the test was finished.");
		}
	}

	resetStreamsAndSendOutput();
	Assert.fail("During the timeout period of " + startTimeoutSeconds + " seconds the " +
			"expected string did not show up");
	return null;
}