Java Code Examples for org.apache.flink.runtime.security.SecurityUtils

The following examples show how to use org.apache.flink.runtime.security.SecurityUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: TestingSecurityContext.java    License: Apache License 2.0 6 votes vote down vote up
public static void install(SecurityConfiguration config,
					Map<String, ClientSecurityConfiguration> clientSecurityConfigurationMap)
		throws Exception {

	SecurityUtils.install(config);

	// install dynamic JAAS entries
	for (SecurityModuleFactory factory : config.getSecurityModuleFactories()) {
		if (factory instanceof JaasModuleFactory) {
			DynamicConfiguration jaasConf = (DynamicConfiguration) javax.security.auth.login.Configuration.getConfiguration();
			for (Map.Entry<String, ClientSecurityConfiguration> e : clientSecurityConfigurationMap.entrySet()) {
				AppConfigurationEntry entry = KerberosUtils.keytabEntry(
					e.getValue().getKeytab(),
					e.getValue().getPrincipal());
				jaasConf.addAppConfigurationEntry(e.getKey(), entry);
			}
			break;
		}
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable);
	}

	System.exit(retCode);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: YarnTaskExecutorRunnerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKerberosKeytabConfiguration() throws Exception {
	final String resourceDirPath = Paths.get("src", "test", "resources").toAbsolutePath().toString();

	final Map<String, String> envs = new HashMap<>(2);
	envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "[email protected]");
	envs.put(YarnConfigKeys.KEYTAB_PATH, resourceDirPath);

	Configuration configuration = new Configuration();
	YarnTaskExecutorRunner.setupConfigurationAndInstallSecurityContext(configuration, resourceDirPath, envs);

	final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
	Optional<SecurityModule> moduleOpt = modules.stream().filter(module -> module instanceof HadoopModule).findFirst();

	if (moduleOpt.isPresent()) {
		HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
		assertThat(hadoopModule.getSecurityConfig().getPrincipal(), is("[email protected]"));
		assertThat(hadoopModule.getSecurityConfig().getKeytab(), is(new File(resourceDirPath, Utils.KEYTAB_FILE_NAME).getAbsolutePath()));
	} else {
		fail("Can not find HadoopModule!");
	}

	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB), is(new File(resourceDirPath, Utils.KEYTAB_FILE_NAME).getAbsolutePath()));
	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), is("[email protected]"));
}
 
Example 4
Source Project: flink   Source File: TestingSecurityContext.java    License: Apache License 2.0 6 votes vote down vote up
public static void install(SecurityConfiguration config,
					Map<String, ClientSecurityConfiguration> clientSecurityConfigurationMap)
		throws Exception {

	SecurityUtils.install(config);

	// install dynamic JAAS entries
	for (SecurityModuleFactory factory : config.getSecurityModuleFactories()) {
		if (factory instanceof JaasModuleFactory) {
			DynamicConfiguration jaasConf = (DynamicConfiguration) javax.security.auth.login.Configuration.getConfiguration();
			for (Map.Entry<String, ClientSecurityConfiguration> e : clientSecurityConfigurationMap.entrySet()) {
				AppConfigurationEntry entry = KerberosUtils.keytabEntry(
					e.getValue().getKeytab(),
					e.getValue().getPrincipal());
				jaasConf.addAppConfigurationEntry(e.getKey(), entry);
			}
			break;
		}
	}
}
 
Example 5
Source Project: flink   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable);
	}

	System.exit(retCode);
}
 
Example 6
Source Project: flink   Source File: YarnTaskExecutorRunnerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKerberosKeytabConfiguration() throws Exception {
	final String resourceDirPath = Paths.get("src", "test", "resources").toAbsolutePath().toString();

	final Map<String, String> envs = new HashMap<>(2);
	envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "[email protected]");
	envs.put(YarnConfigKeys.KEYTAB_PATH, resourceDirPath);

	Configuration configuration = new Configuration();
	YarnTaskExecutorRunner.setupConfigurationAndInstallSecurityContext(configuration, resourceDirPath, envs);

	final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
	Optional<SecurityModule> moduleOpt = modules.stream().filter(module -> module instanceof HadoopModule).findFirst();

	if (moduleOpt.isPresent()) {
		HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
		assertThat(hadoopModule.getSecurityConfig().getPrincipal(), is("[email protected]"));
		assertThat(hadoopModule.getSecurityConfig().getKeytab(), is(new File(resourceDirPath, Utils.KEYTAB_FILE_NAME).getAbsolutePath()));
	} else {
		fail("Can not find HadoopModule!");
	}

	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB), is(new File(resourceDirPath, Utils.KEYTAB_FILE_NAME).getAbsolutePath()));
	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), is("[email protected]"));
}
 
Example 7
Source Project: flink   Source File: TestingSecurityContext.java    License: Apache License 2.0 6 votes vote down vote up
public static void install(SecurityConfiguration config,
					Map<String, ClientSecurityConfiguration> clientSecurityConfigurationMap)
		throws Exception {

	SecurityUtils.install(config);

	// install dynamic JAAS entries
	for (String factoryClassName : config.getSecurityModuleFactories()) {
		SecurityModuleFactory factory = SecurityFactoryServiceLoader.findModuleFactory(factoryClassName);
		if (factory instanceof JaasModuleFactory) {
			DynamicConfiguration jaasConf = (DynamicConfiguration) javax.security.auth.login.Configuration.getConfiguration();
			for (Map.Entry<String, ClientSecurityConfiguration> e : clientSecurityConfigurationMap.entrySet()) {
				AppConfigurationEntry entry = KerberosUtils.keytabEntry(
					e.getValue().getKeytab(),
					e.getValue().getPrincipal());
				jaasConf.addAppConfigurationEntry(e.getKey(), entry);
			}
			break;
		}
	}
}
 
Example 8
Source Project: flink   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e, LOG);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable, LOG);
	}

	System.exit(retCode);
}
 
Example 9
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 6 votes vote down vote up
public static void runTaskManagerSecurely(String[] args) {
	try {
		final Configuration configuration = loadConfiguration(args);
		final PluginManager pluginManager = PluginUtils.createPluginManagerFromRootFolder(configuration);
		FileSystem.initialize(configuration, pluginManager);

		SecurityUtils.install(new SecurityConfiguration(configuration));

		SecurityUtils.getInstalledContext().runSecured(() -> {
			runTaskManager(configuration, pluginManager);
			return null;
		});
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("TaskManager initialization failed.", strippedThrowable);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
	}
}
 
Example 10
Source Project: Flink-CEPplus   Source File: HistoryServer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);
	String configDir = pt.getRequired("configDir");

	LOG.info("Loading configuration from {}", configDir);
	final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(configDir);

	try {
		FileSystem.initialize(flinkConfig);
	} catch (IOException e) {
		throw new Exception("Error while setting the default filesystem scheme from configuration.", e);
	}

	// run the history server
	SecurityUtils.install(new SecurityConfiguration(flinkConfig));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Integer>() {
			@Override
			public Integer call() throws Exception {
				HistoryServer hs = new HistoryServer(flinkConfig);
				hs.run();
				return 0;
			}
		});
		System.exit(0);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Failed to run HistoryServer.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(1);
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The instance entry point for the YARN task executor. Obtains user group information and calls
 * the main work method {@link TaskManagerRunner#runTaskManager(Configuration, ResourceID)}  as a
 * privileged action.
 *
 * @param args The command line arguments.
 */
private static void run(String[] args) {
	try {
		LOG.debug("All environment variables: {}", ENV);

		final String currDir = ENV.get(Environment.PWD.key());
		LOG.info("Current working Directory: {}", currDir);

		final Configuration configuration = GlobalConfiguration.loadConfiguration(currDir);
		FileSystem.initialize(configuration);

		setupConfigurationAndInstallSecurityContext(configuration, currDir, ENV);

		final String containerId = ENV.get(YarnResourceManager.ENV_FLINK_CONTAINER_ID);
		Preconditions.checkArgument(containerId != null,
			"ContainerId variable %s not set", YarnResourceManager.ENV_FLINK_CONTAINER_ID);

		SecurityUtils.getInstalledContext().runSecured((Callable<Void>) () -> {
			TaskManagerRunner.runTaskManager(configuration, new ResourceID(containerId));
			return null;
		});
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		// make sure that everything whatever ends up in the log
		LOG.error("YARN TaskManager initialization failed.", strippedThrowable);
		System.exit(INIT_ERROR_EXIT_CODE);
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: TaskManagerRunner.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager", args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	long maxOpenFileHandles = EnvironmentInformation.getOpenFileHandlesLimit();

	if (maxOpenFileHandles != -1L) {
		LOG.info("Maximum number of open file descriptors is {}.", maxOpenFileHandles);
	} else {
		LOG.info("Cannot determine the maximum number of open file descriptors");
	}

	final Configuration configuration = loadConfiguration(args);

	try {
		FileSystem.initialize(configuration);
	} catch (IOException e) {
		throw new IOException("Error while setting the default " +
			"filesystem scheme from configuration.", e);
	}

	SecurityUtils.install(new SecurityConfiguration(configuration));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Void>() {
			@Override
			public Void call() throws Exception {
				runTaskManager(configuration, ResourceID.generate());
				return null;
			}
		});
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("TaskManager initialization failed.", strippedThrowable);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: CliFrontend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Submits the job based on the arguments.
 */
public static void main(final String[] args) {
	EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

	// 1. find the configuration directory
	final String configurationDirectory = getConfigurationDirectoryFromEnv();

	// 2. load the global configuration
	final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);

	// 3. load the custom command lines
	final List<CustomCommandLine<?>> customCommandLines = loadCustomCommandLines(
		configuration,
		configurationDirectory);

	try {
		final CliFrontend cli = new CliFrontend(
			configuration,
			customCommandLines);

		SecurityUtils.install(new SecurityConfiguration(cli.configuration));
		int retCode = SecurityUtils.getInstalledContext()
				.runSecured(() -> cli.parseParameters(args));
		System.exit(retCode);
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Fatal error while running command line interface.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(31);
	}
}
 
Example 14
Source Project: flink   Source File: HistoryServer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);
	String configDir = pt.getRequired("configDir");

	LOG.info("Loading configuration from {}", configDir);
	final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(configDir);

	FileSystem.initialize(flinkConfig, PluginUtils.createPluginManagerFromRootFolder(flinkConfig));

	// run the history server
	SecurityUtils.install(new SecurityConfiguration(flinkConfig));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Integer>() {
			@Override
			public Integer call() throws Exception {
				HistoryServer hs = new HistoryServer(flinkConfig);
				hs.run();
				return 0;
			}
		});
		System.exit(0);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Failed to run HistoryServer.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(1);
	}
}
 
Example 15
Source Project: flink   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The instance entry point for the YARN task executor. Obtains user group information and calls
 * the main work method {@link TaskManagerRunner#runTaskManager(Configuration, ResourceID)}  as a
 * privileged action.
 *
 * @param args The command line arguments.
 */
private static void run(String[] args) {
	try {
		LOG.debug("All environment variables: {}", ENV);

		final String currDir = ENV.get(Environment.PWD.key());
		LOG.info("Current working Directory: {}", currDir);

		final Configuration configuration = GlobalConfiguration.loadConfiguration(currDir);

		FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

		setupConfigurationAndInstallSecurityContext(configuration, currDir, ENV);

		final String containerId = ENV.get(YarnResourceManager.ENV_FLINK_CONTAINER_ID);
		Preconditions.checkArgument(containerId != null,
			"ContainerId variable %s not set", YarnResourceManager.ENV_FLINK_CONTAINER_ID);

		SecurityUtils.getInstalledContext().runSecured((Callable<Void>) () -> {
			TaskManagerRunner.runTaskManager(configuration, new ResourceID(containerId));
			return null;
		});
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		// make sure that everything whatever ends up in the log
		LOG.error("YARN TaskManager initialization failed.", strippedThrowable);
		System.exit(INIT_ERROR_EXIT_CODE);
	}
}
 
Example 16
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager", args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	long maxOpenFileHandles = EnvironmentInformation.getOpenFileHandlesLimit();

	if (maxOpenFileHandles != -1L) {
		LOG.info("Maximum number of open file descriptors is {}.", maxOpenFileHandles);
	} else {
		LOG.info("Cannot determine the maximum number of open file descriptors");
	}

	final Configuration configuration = loadConfiguration(args);

	FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

	SecurityUtils.install(new SecurityConfiguration(configuration));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Void>() {
			@Override
			public Void call() throws Exception {
				runTaskManager(configuration, ResourceID.generate());
				return null;
			}
		});
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("TaskManager initialization failed.", strippedThrowable);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
	}
}
 
Example 17
Source Project: flink   Source File: CliFrontend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Submits the job based on the arguments.
 */
public static void main(final String[] args) {
	EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

	// 1. find the configuration directory
	final String configurationDirectory = getConfigurationDirectoryFromEnv();

	// 2. load the global configuration
	final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);

	// 3. load the custom command lines
	final List<CustomCommandLine<?>> customCommandLines = loadCustomCommandLines(
		configuration,
		configurationDirectory);

	try {
		final CliFrontend cli = new CliFrontend(
			configuration,
			customCommandLines);

		SecurityUtils.install(new SecurityConfiguration(cli.configuration));
		int retCode = SecurityUtils.getInstalledContext()
				.runSecured(() -> cli.parseParameters(args));
		System.exit(retCode);
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Fatal error while running command line interface.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(31);
	}
}
 
Example 18
Source Project: flink   Source File: HistoryServer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	EnvironmentInformation.logEnvironmentInfo(LOG, "HistoryServer", args);

	ParameterTool pt = ParameterTool.fromArgs(args);
	String configDir = pt.getRequired("configDir");

	LOG.info("Loading configuration from {}", configDir);
	final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(configDir);

	FileSystem.initialize(flinkConfig, PluginUtils.createPluginManagerFromRootFolder(flinkConfig));

	// run the history server
	SecurityUtils.install(new SecurityConfiguration(flinkConfig));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Integer>() {
			@Override
			public Integer call() throws Exception {
				HistoryServer hs = new HistoryServer(flinkConfig);
				hs.run();
				return 0;
			}
		});
		System.exit(0);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Failed to run HistoryServer.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(1);
	}
}
 
Example 19
Source Project: flink   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The instance entry point for the YARN task executor. Obtains user group information and calls
 * the main work method {@link TaskManagerRunner#runTaskManager(Configuration, PluginManager)} as a
 * privileged action.
 *
 * @param args The command line arguments.
 */
private static void runTaskManagerSecurely(String[] args) {
	try {
		LOG.debug("All environment variables: {}", ENV);

		final String currDir = ENV.get(Environment.PWD.key());
		LOG.info("Current working Directory: {}", currDir);

		final Configuration configuration = TaskManagerRunner.loadConfiguration(args);

		final PluginManager pluginManager = PluginUtils.createPluginManagerFromRootFolder(configuration);

		FileSystem.initialize(configuration, pluginManager);

		setupConfigurationAndInstallSecurityContext(configuration, currDir, ENV);

		SecurityUtils.getInstalledContext().runSecured((Callable<Void>) () -> {
			TaskManagerRunner.runTaskManager(configuration, pluginManager);
			return null;
		});
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		// make sure that everything whatever ends up in the log
		LOG.error("YARN TaskManager initialization failed.", strippedThrowable);
		System.exit(INIT_ERROR_EXIT_CODE);
	}
}
 
Example 20
Source Project: flink   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static void setupConfigurationAndInstallSecurityContext(Configuration configuration, String currDir, Map<String, String> variables) throws Exception {
	final String localDirs = variables.get(Environment.LOCAL_DIRS.key());
	LOG.info("Current working/local Directory: {}", localDirs);

	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

	setupConfigurationFromVariables(configuration, currDir, variables);

	SecurityUtils.install(new SecurityConfiguration(configuration));
}
 
Example 21
Source Project: flink   Source File: YarnTaskExecutorRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultKerberosKeytabConfiguration() throws Exception {
	final String resourceDirPath = Paths.get("src", "test", "resources").toAbsolutePath().toString();

	final Map<String, String> envs = new HashMap<>(2);
	envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "[email protected]");
	envs.put(YarnConfigKeys.REMOTE_KEYTAB_PATH, resourceDirPath);
	// Local keytab path will be populated from default YarnConfigOptions.LOCALIZED_KEYTAB_PATH
	envs.put(YarnConfigKeys.LOCAL_KEYTAB_PATH, YarnConfigOptions.LOCALIZED_KEYTAB_PATH.defaultValue());

	Configuration configuration = new Configuration();
	YarnTaskExecutorRunner.setupConfigurationAndInstallSecurityContext(configuration, resourceDirPath, envs);

	final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
	Optional<SecurityModule> moduleOpt = modules.stream().filter(module -> module instanceof HadoopModule).findFirst();

	if (moduleOpt.isPresent()) {
		HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
		assertThat(hadoopModule.getSecurityConfig().getPrincipal(), is("[email protected]"));
		assertThat(hadoopModule.getSecurityConfig().getKeytab(), is(new File(resourceDirPath, YarnConfigOptions.LOCALIZED_KEYTAB_PATH.defaultValue()).getAbsolutePath()));
	} else {
		fail("Can not find HadoopModule!");
	}

	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB), is(new File(resourceDirPath, YarnConfigOptions.LOCALIZED_KEYTAB_PATH.defaultValue()).getAbsolutePath()));
	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), is("[email protected]"));
}
 
Example 22
Source Project: flink   Source File: YarnTaskExecutorRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreInstallKerberosKeytabConfiguration() throws Exception {
	final String resourceDirPath = Paths.get("src", "test", "resources").toAbsolutePath().toString();

	final Map<String, String> envs = new HashMap<>(2);
	envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "[email protected]");
	// Try directly resolving local path when no remote keytab path is provided.
	envs.put(YarnConfigKeys.LOCAL_KEYTAB_PATH, "src/test/resources/krb5.keytab");

	Configuration configuration = new Configuration();
	YarnTaskExecutorRunner.setupConfigurationAndInstallSecurityContext(configuration, resourceDirPath, envs);

	final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
	Optional<SecurityModule> moduleOpt = modules.stream().filter(module -> module instanceof HadoopModule).findFirst();

	if (moduleOpt.isPresent()) {
		HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
		assertThat(hadoopModule.getSecurityConfig().getPrincipal(), is("[email protected]"));
		// Using containString verification as the absolute path varies depending on runtime environment
		assertThat(hadoopModule.getSecurityConfig().getKeytab(), containsString("src/test/resources/krb5.keytab"));
	} else {
		fail("Can not find HadoopModule!");
	}

	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB), containsString("src/test/resources/krb5.keytab"));
	assertThat(configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), is("[email protected]"));

}
 
Example 23
Source Project: flink   Source File: CliFrontend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Submits the job based on the arguments.
 */
public static void main(final String[] args) {
	EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

	// 1. find the configuration directory
	final String configurationDirectory = getConfigurationDirectoryFromEnv();

	// 2. load the global configuration
	final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);

	// 3. load the custom command lines
	final List<CustomCommandLine> customCommandLines = loadCustomCommandLines(
		configuration,
		configurationDirectory);

	try {
		final CliFrontend cli = new CliFrontend(
			configuration,
			customCommandLines);

		SecurityUtils.install(new SecurityConfiguration(cli.configuration));
		int retCode = SecurityUtils.getInstalledContext()
				.runSecured(() -> cli.parseParameters(args));
		System.exit(retCode);
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Fatal error while running command line interface.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(31);
	}
}
 
Example 24
Source Project: Flink-CEPplus   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 4 votes vote down vote up
private static void installSecurityContext(Configuration configuration) throws Exception {
	SecurityConfiguration sc = new SecurityConfiguration(configuration);
	SecurityUtils.install(sc);
}
 
Example 25
@BeforeClass
public static void setup() {

	LOG.info("starting secure cluster environment for testing");

	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo-secured");

	SecureTestEnvironment.prepare(tmp);

	populateYarnSecureConfigurations(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
			SecureTestEnvironment.getTestKeytab());

	Configuration flinkConfig = new Configuration();
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB,
			SecureTestEnvironment.getTestKeytab());
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
			SecureTestEnvironment.getHadoopServicePrincipal());

	SecurityConfiguration securityConfig =
		new SecurityConfiguration(
			flinkConfig,
			Collections.singletonList(securityConfig1 -> {
				// manually override the Hadoop Configuration
				return new HadoopModule(securityConfig1, YARN_CONFIGURATION);
			}));

	try {
		TestingSecurityContext.install(securityConfig, SecureTestEnvironment.getClientSecurityConfigurationMap());

		SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
			@Override
			public Integer call() {
				startYARNSecureMode(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
						SecureTestEnvironment.getTestKeytab());
				return null;
			}
		});

	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
	}

}
 
Example 26
Source Project: flink   Source File: YarnTaskExecutorRunner.java    License: Apache License 2.0 4 votes vote down vote up
private static void installSecurityContext(Configuration configuration) throws Exception {
	SecurityConfiguration sc = new SecurityConfiguration(configuration);
	SecurityUtils.install(sc);
}
 
Example 27
Source Project: flink   Source File: YARNSessionFIFOSecuredITCase.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() {

	LOG.info("starting secure cluster environment for testing");

	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo-secured");

	SecureTestEnvironment.prepare(tmp);

	populateYarnSecureConfigurations(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
			SecureTestEnvironment.getTestKeytab());

	Configuration flinkConfig = new Configuration();
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB,
			SecureTestEnvironment.getTestKeytab());
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
			SecureTestEnvironment.getHadoopServicePrincipal());

	SecurityConfiguration securityConfig =
		new SecurityConfiguration(
			flinkConfig,
			Collections.singletonList(securityConfig1 -> {
				// manually override the Hadoop Configuration
				return new HadoopModule(securityConfig1, YARN_CONFIGURATION);
			}));

	try {
		TestingSecurityContext.install(securityConfig, SecureTestEnvironment.getClientSecurityConfigurationMap());

		SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
			@Override
			public Integer call() {
				startYARNSecureMode(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
						SecureTestEnvironment.getTestKeytab());
				return null;
			}
		});

	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
	}

}
 
Example 28
Source Project: flink   Source File: YARNSessionFIFOSecuredITCase.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() {

	LOG.info("starting secure cluster environment for testing");

	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo-secured");

	SecureTestEnvironment.prepare(tmp);

	populateYarnSecureConfigurations(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
			SecureTestEnvironment.getTestKeytab());

	Configuration flinkConfig = new Configuration();
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB,
		SecureTestEnvironment.getTestKeytab());
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
		SecureTestEnvironment.getHadoopServicePrincipal());

	// Setting customized security module class.
	TestHadoopModuleFactory.hadoopConfiguration = YARN_CONFIGURATION;
	flinkConfig.set(SecurityOptions.SECURITY_MODULE_FACTORY_CLASSES,
		Collections.singletonList("org.apache.flink.yarn.util.TestHadoopModuleFactory"));
	flinkConfig.set(SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
		Collections.singletonList("org.apache.flink.yarn.util.TestHadoopSecurityContextFactory"));

	SecurityConfiguration securityConfig =
		new SecurityConfiguration(flinkConfig);

	try {
		TestingSecurityContext.install(securityConfig, SecureTestEnvironment.getClientSecurityConfigurationMap());

		// This is needed to ensure that SecurityUtils are run within a ugi.doAs section
		// Since we already logged in here in @BeforeClass, even a no-op security context will still work.
		Assert.assertTrue("HadoopSecurityContext must be installed",
			SecurityUtils.getInstalledContext() instanceof HadoopSecurityContext);
		SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
			@Override
			public Integer call() {
				startYARNSecureMode(YARN_CONFIGURATION, SecureTestEnvironment.getHadoopServicePrincipal(),
						SecureTestEnvironment.getTestKeytab());
				return null;
			}
		});

	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
	}

}
 
Example 29
Source Project: Flink-CEPplus   Source File: YarnEntrypointUtils.java    License: Apache License 2.0 3 votes vote down vote up
public static SecurityContext installSecurityContext(
		Configuration configuration,
		String workingDirectory) throws Exception {

	SecurityConfiguration sc = new SecurityConfiguration(configuration);

	SecurityUtils.install(sc);

	return SecurityUtils.getInstalledContext();
}
 
Example 30
Source Project: Flink-CEPplus   Source File: ClusterEntrypoint.java    License: Apache License 2.0 3 votes vote down vote up
protected SecurityContext installSecurityContext(Configuration configuration) throws Exception {
	LOG.info("Install security context.");

	SecurityUtils.install(new SecurityConfiguration(configuration));

	return SecurityUtils.getInstalledContext();
}