org.apache.flink.client.cli.CustomCommandLine Java Examples

/**
 * Constructor for testing purposes.
 */
public LocalExecutor(
		Environment defaultEnvironment,
		List<URL> dependencies,
		Configuration flinkConfig,
		CustomCommandLine commandLine,
		ClusterClientServiceLoader clusterClientServiceLoader) {
	this.defaultEnvironment = defaultEnvironment;
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;
	this.commandLines = Collections.singletonList(commandLine);
	this.commandLineOptions = collectCommandLineOptions(commandLines);
	this.contextMap = new ConcurrentHashMap<>();

	// prepare result store
	this.resultStore = new ResultStore(flinkConfig);
	this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader);
}
 

private Builder(
		Environment defaultEnv,
		@Nullable SessionContext sessionContext,
		List<URL> dependencies,
		Configuration configuration,
		ClusterClientServiceLoader serviceLoader,
		Options commandLineOptions,
		List<CustomCommandLine> commandLines) {
	this.defaultEnv = defaultEnv;
	this.sessionContext = sessionContext;
	this.dependencies = dependencies;
	this.configuration = configuration;
	this.serviceLoader = serviceLoader;
	this.commandLineOptions = commandLineOptions;
	this.commandLines = commandLines;
}
 

private static Options collectCommandLineOptions(List<CustomCommandLine<?>> commandLines) {
	final Options customOptions = new Options();
	for (CustomCommandLine<?> customCommandLine : commandLines) {
		customCommandLine.addRunOptions(customOptions);
	}
	return CliFrontendParser.mergeOptions(
		CliFrontendParser.getRunCommandOptions(),
		customOptions);
}
 

private void validateExecutorCLIisPrioritised(Configuration configuration, String[] argsUnderTest) throws IOException, CliArgsException {
	final List<CustomCommandLine> customCommandLines = CliFrontend.loadCustomCommandLines(
			configuration,
			tmp.newFile().getAbsolutePath());

	final CliFrontend cli = new CliFrontend(configuration, customCommandLines);
	final CommandLine commandLine = cli.getCommandLine(
			CliFrontendParser.getRunCommandOptions(),
			argsUnderTest,
			true);

	final CustomCommandLine customCommandLine = cli.validateAndGetActiveCommandLine(commandLine);
	assertTrue(customCommandLine instanceof GenericCLI);
}
 

private static Options collectCommandLineOptions(List<CustomCommandLine> commandLines) {
	final Options customOptions = new Options();
	for (CustomCommandLine customCommandLine : commandLines) {
		customCommandLine.addGeneralOptions(customOptions);
		customCommandLine.addRunOptions(customOptions);
	}
	return CliFrontendParser.mergeOptions(
		CliFrontendParser.getRunCommandOptions(),
		customOptions);
}
 

private static CustomCommandLine findActiveCommandLine(List<CustomCommandLine> availableCommandLines, CommandLine commandLine) {
	for (CustomCommandLine cli : availableCommandLines) {
		if (cli.isActive(commandLine)) {
			return cli;
		}
	}
	throw new SqlExecutionException("Could not find a matching deployment.");
}
 

private static Configuration createExecutionConfig(
		CommandLine commandLine,
		Options commandLineOptions,
		List<CustomCommandLine> availableCommandLines,
		List<URL> dependencies) throws FlinkException {
	LOG.debug("Available commandline options: {}", commandLineOptions);
	List<String> options = Stream
			.of(commandLine.getOptions())
			.map(o -> o.getOpt() + "=" + o.getValue())
			.collect(Collectors.toList());
	LOG.debug(
			"Instantiated commandline args: {}, options: {}",
			commandLine.getArgList(),
			options);

	final CustomCommandLine activeCommandLine = findActiveCommandLine(
			availableCommandLines,
			commandLine);
	LOG.debug(
			"Available commandlines: {}, active commandline: {}",
			availableCommandLines,
			activeCommandLine);

	Configuration executionConfig = activeCommandLine.applyCommandLineOptionsToConfiguration(
			commandLine);

	try {
		final ProgramOptions programOptions = ProgramOptions.create(commandLine);
		final ExecutionConfigAccessor executionConfigAccessor = ExecutionConfigAccessor.fromProgramOptions(programOptions, dependencies);
		executionConfigAccessor.applyToConfiguration(executionConfig);
	} catch (CliArgsException e) {
		throw new SqlExecutionException("Invalid deployment run options.", e);
	}

	LOG.info("Executor config: {}", executionConfig);
	return executionConfig;
}
 

/** Returns a builder for this {@link ExecutionContext}. */
public static Builder builder(
		Environment defaultEnv,
		SessionContext sessionContext,
		List<URL> dependencies,
		Configuration configuration,
		ClusterClientServiceLoader serviceLoader,
		Options commandLineOptions,
		List<CustomCommandLine> commandLines) {
	return new Builder(defaultEnv, sessionContext, dependencies, configuration,
			serviceLoader, commandLineOptions, commandLines);
}
 

public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create table sources & sinks.
	tableSources = new HashMap<>();
	tableSinks = new HashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new HashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}
 

/**
 * Constructor for testing purposes.
 */
public LocalExecutor(Environment defaultEnvironment, List<URL> dependencies, Configuration flinkConfig, CustomCommandLine<?> commandLine) {
	this.defaultEnvironment = defaultEnvironment;
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;
	this.commandLines = Collections.singletonList(commandLine);
	this.commandLineOptions = collectCommandLineOptions(commandLines);

	// prepare result store
	resultStore = new ResultStore(flinkConfig);
}
 

private static ClusterSpecification createClusterSpecification(CustomCommandLine<?> activeCommandLine, CommandLine commandLine) {
	try {
		return activeCommandLine.getClusterSpecification(commandLine);
	} catch (FlinkException e) {
		throw new SqlExecutionException("Could not create cluster specification for the given deployment.", e);
	}
}
 

@SuppressWarnings("unchecked")
private static <T> CustomCommandLine<T> findActiveCommandLine(List<CustomCommandLine<?>> availableCommandLines, CommandLine commandLine) {
	for (CustomCommandLine<?> cli : availableCommandLines) {
		if (cli.isActive(commandLine)) {
			return (CustomCommandLine<T>) cli;
		}
	}
	throw new SqlExecutionException("Could not find a matching deployment.");
}
 

private static Options collectCommandLineOptions(List<CustomCommandLine<?>> commandLines) {
	final Options customOptions = new Options();
	for (CustomCommandLine<?> customCommandLine : commandLines) {
		customCommandLine.addRunOptions(customOptions);
	}
	return CliFrontendParser.mergeOptions(
		CliFrontendParser.getRunCommandOptions(),
		customOptions);
}
 

/**
 * Constructor for testing purposes.
 */
public LocalExecutor(Environment defaultEnvironment, List<URL> dependencies, Configuration flinkConfig, CustomCommandLine<?> commandLine) {
	this.defaultEnvironment = defaultEnvironment;
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;
	this.commandLines = Collections.singletonList(commandLine);
	this.commandLineOptions = collectCommandLineOptions(commandLines);

	// prepare result store
	resultStore = new ResultStore(flinkConfig);
}
 

private static ClusterSpecification createClusterSpecification(CustomCommandLine<?> activeCommandLine, CommandLine commandLine) {
	try {
		return activeCommandLine.getClusterSpecification(commandLine);
	} catch (FlinkException e) {
		throw new SqlExecutionException("Could not create cluster specification for the given deployment.", e);
	}
}
 

@SuppressWarnings("unchecked")
private static <T> CustomCommandLine<T> findActiveCommandLine(List<CustomCommandLine<?>> availableCommandLines, CommandLine commandLine) {
	for (CustomCommandLine<?> cli : availableCommandLines) {
		if (cli.isActive(commandLine)) {
			return (CustomCommandLine<T>) cli;
		}
	}
	throw new SqlExecutionException("Could not find a matching deployment.");
}
 

private ExecutionContext(
		Environment environment,
		SessionContext originalSessionContext,
		@Nullable SessionState sessionState,
		List<URL> dependencies,
		Configuration flinkConfig,
		ClusterClientServiceLoader clusterClientServiceLoader,
		Options commandLineOptions,
		List<CustomCommandLine> availableCommandLines) throws FlinkException {
	this.environment = environment;
	this.originalSessionContext = originalSessionContext;

	this.flinkConfig = flinkConfig;

	if (containsPythonFunction(environment)) {
		dependencies = addPythonDependency(dependencies);
	}

	// create class loader
	classLoader = ClientUtils.buildUserCodeClassLoader(
		dependencies,
		Collections.emptyList(),
		this.getClass().getClassLoader(),
		flinkConfig);

	// Initialize the TableEnvironment.
	initializeTableEnvironment(sessionState);

	LOG.debug("Deployment descriptor: {}", environment.getDeployment());
	final CommandLine commandLine = createCommandLine(
			environment.getDeployment(),
			commandLineOptions);

	flinkConfig.addAll(createExecutionConfig(
			commandLine,
			commandLineOptions,
			availableCommandLines,
			dependencies));

	final ClusterClientServiceLoader serviceLoader = checkNotNull(clusterClientServiceLoader);
	clusterClientFactory = serviceLoader.getClusterClientFactory(flinkConfig);
	checkState(clusterClientFactory != null);

	clusterId = clusterClientFactory.getClusterId(flinkConfig);
	clusterSpec = clusterClientFactory.getClusterSpecification(flinkConfig);
}
 

public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create catalogs
	catalogs = new LinkedHashMap<>();
	mergedEnv.getCatalogs().forEach((name, entry) ->
		catalogs.put(name, createCatalog(name, entry.asMap(), classLoader))
	);

	// create table sources & sinks.
	tableSources = new LinkedHashMap<>();
	tableSinks = new LinkedHashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new LinkedHashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}