Java Code Examples for org.apache.hadoop.security.Credentials#writeTokenStorageFile()

The following examples show how to use org.apache.hadoop.security.Credentials#writeTokenStorageFile() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestTokenCache.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testGetTokensForNamenodes() throws IOException,
    URISyntaxException {
  Path TEST_ROOT_DIR =
      new Path(System.getProperty("test.build.data", "test/build/data"));
  // ick, but need fq path minus file:/
  String binaryTokenFile =
      FileSystem.getLocal(conf)
        .makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri()
        .getPath();

  MockFileSystem fs1 = createFileSystemForServiceName("service1");
  Credentials creds = new Credentials();
  Token<?> token1 = fs1.getDelegationToken(renewer);
  creds.addToken(token1.getService(), token1);
  // wait to set, else the obtain tokens call above will fail with FNF
  conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
  creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
  TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
  String fs_addr = fs1.getCanonicalServiceName();
  Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
  assertNotNull("Token for nn is null", nnt);
}
 
Example 2
Source File: TestTokenCache.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testGetTokensForNamenodes() throws IOException,
    URISyntaxException {
  Path TEST_ROOT_DIR =
      new Path(System.getProperty("test.build.data", "test/build/data"));
  // ick, but need fq path minus file:/
  String binaryTokenFile =
      FileSystem.getLocal(conf)
        .makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri()
        .getPath();

  MockFileSystem fs1 = createFileSystemForServiceName("service1");
  Credentials creds = new Credentials();
  Token<?> token1 = fs1.getDelegationToken(renewer);
  creds.addToken(token1.getService(), token1);
  // wait to set, else the obtain tokens call above will fail with FNF
  conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
  creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
  TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
  String fs_addr = fs1.getCanonicalServiceName();
  Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
  assertNotNull("Token for nn is null", nnt);
}
 
Example 3
Source File: TestTokenCache.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("deprecation")
public void testBinaryCredentials() throws Exception {
  String binaryTokenFile = null;
  try {
    Path TEST_ROOT_DIR = new Path("target");
    binaryTokenFile = FileSystem.getLocal(conf).makeQualified(
      new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();

    MockFileSystem fs1 = createFileSystemForServiceName("service1");
    MockFileSystem fs2 = createFileSystemForServiceName("service2");
    MockFileSystem fs3 = createFileSystemForServiceName("service3");

    // get the tokens for fs1 & fs2 and write out to binary creds file
    Credentials creds = new Credentials();
    Token<?> token1 = fs1.getDelegationToken(renewer);
    Token<?> token2 = fs2.getDelegationToken(renewer);
    creds.addToken(token1.getService(), token1);
    creds.addToken(token2.getService(), token2);
    creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);


    Credentials newCreds = new Credentials();
    TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile);

    Assert.assertTrue(newCreds.getAllTokens().size() > 0);
    checkTokens(creds, newCreds);
  } finally {
    if (binaryTokenFile != null) {
      try {
        FileSystem.getLocal(conf).delete(new Path(binaryTokenFile));
      } catch (IOException e) {
        // Ignore
      }
    }
  }
}
 
Example 4
Source File: TestTokenCache.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
@SuppressWarnings("deprecation")
public void testBinaryCredentials() throws Exception {
  String binaryTokenFile = null;
  try {
    Path TEST_ROOT_DIR = new Path("target");
    binaryTokenFile = FileSystem.getLocal(conf).makeQualified(
      new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();

    MockFileSystem fs1 = createFileSystemForServiceName("service1");
    MockFileSystem fs2 = createFileSystemForServiceName("service2");

    // get the tokens for fs1 & fs2 and write out to binary creds file
    Credentials creds = new Credentials();
    Token<?> token1 = fs1.getDelegationToken(renewer);
    Token<?> token2 = fs2.getDelegationToken(renewer);
    creds.addToken(token1.getService(), token1);
    creds.addToken(token2.getService(), token2);
    creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);


    Credentials newCreds = new Credentials();
    TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile);

    Assert.assertTrue(newCreds.getAllTokens().size() > 0);
    checkTokens(creds, newCreds);
  } finally {
    if (binaryTokenFile != null) {
      try {
        FileSystem.getLocal(conf).delete(new Path(binaryTokenFile));
      } catch (IOException e) {
        // Ignore
      }
    }
  }
}
 
Example 5
Source File: UtilsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testCreateTaskExecutorCredentials() throws Exception {
	File root = temporaryFolder.getRoot();
	File home = new File(root, "home");
	boolean created = home.mkdir();
	assertTrue(created);

	Configuration flinkConf = new Configuration();
	YarnConfiguration yarnConf = new YarnConfiguration();

	Map<String, String> env = new HashMap<>();
	env.put(YarnConfigKeys.ENV_APP_ID, "foo");
	env.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, home.getAbsolutePath());
	env.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, "");
	env.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, "");
	env.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, "foo");
	env.put(YarnConfigKeys.FLINK_JAR_PATH, root.toURI().toString());
	env = Collections.unmodifiableMap(env);

	File credentialFile = temporaryFolder.newFile("container_tokens");
	final Text amRmTokenKind = AMRMTokenIdentifier.KIND_NAME;
	final Text hdfsDelegationTokenKind = new Text("HDFS_DELEGATION_TOKEN");
	final Text service = new Text("test-service");
	Credentials amCredentials = new Credentials();
	amCredentials.addToken(amRmTokenKind, new Token<>(new byte[4], new byte[4], amRmTokenKind, service));
	amCredentials.addToken(hdfsDelegationTokenKind, new Token<>(new byte[4], new byte[4],
		hdfsDelegationTokenKind, service));
	amCredentials.writeTokenStorageFile(new org.apache.hadoop.fs.Path(credentialFile.getAbsolutePath()), yarnConf);

	ContaineredTaskManagerParameters tmParams = new ContaineredTaskManagerParameters(64,
		64, 16, 1, new HashMap<>(1));
	Configuration taskManagerConf = new Configuration();

	String workingDirectory = root.getAbsolutePath();
	Class<?> taskManagerMainClass = YarnTaskExecutorRunner.class;
	ContainerLaunchContext ctx;

	final Map<String, String> originalEnv = System.getenv();
	try {
		Map<String, String> systemEnv = new HashMap<>(originalEnv);
		systemEnv.put("HADOOP_TOKEN_FILE_LOCATION", credentialFile.getAbsolutePath());
		CommonTestUtils.setEnv(systemEnv);
		ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams,
			taskManagerConf, workingDirectory, taskManagerMainClass, LOG);
	} finally {
		CommonTestUtils.setEnv(originalEnv);
	}

	Credentials credentials = new Credentials();
	try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(ctx.getTokens().array()))) {
		credentials.readTokenStorageStream(dis);
	}
	Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens();
	boolean hasHdfsDelegationToken = false;
	boolean hasAmRmToken = false;
	for (Token<? extends TokenIdentifier> token : tokens) {
		if (token.getKind().equals(amRmTokenKind)) {
			hasAmRmToken = true;
		} else if (token.getKind().equals(hdfsDelegationTokenKind)) {
			hasHdfsDelegationToken = true;
		}
	}
	assertTrue(hasHdfsDelegationToken);
	assertFalse(hasAmRmToken);
}
 
Example 6
Source File: UtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testCreateTaskExecutorCredentials() throws Exception {
	File root = temporaryFolder.getRoot();
	File home = new File(root, "home");
	boolean created = home.mkdir();
	assertTrue(created);

	Configuration flinkConf = new Configuration();
	YarnConfiguration yarnConf = new YarnConfiguration();

	Map<String, String> env = new HashMap<>();
	env.put(YarnConfigKeys.ENV_APP_ID, "foo");
	env.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, home.getAbsolutePath());
	env.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, "");
	env.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, "");
	env.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, "foo");
	env.put(YarnConfigKeys.FLINK_JAR_PATH, root.toURI().toString());
	env = Collections.unmodifiableMap(env);

	File credentialFile = temporaryFolder.newFile("container_tokens");
	final Text amRmTokenKind = AMRMTokenIdentifier.KIND_NAME;
	final Text hdfsDelegationTokenKind = new Text("HDFS_DELEGATION_TOKEN");
	final Text service = new Text("test-service");
	Credentials amCredentials = new Credentials();
	amCredentials.addToken(amRmTokenKind, new Token<>(new byte[4], new byte[4], amRmTokenKind, service));
	amCredentials.addToken(hdfsDelegationTokenKind, new Token<>(new byte[4], new byte[4],
		hdfsDelegationTokenKind, service));
	amCredentials.writeTokenStorageFile(new org.apache.hadoop.fs.Path(credentialFile.getAbsolutePath()), yarnConf);

	ContaineredTaskManagerParameters tmParams = new ContaineredTaskManagerParameters(64,
		64, 16, 1, new HashMap<>(1));
	Configuration taskManagerConf = new Configuration();

	String workingDirectory = root.getAbsolutePath();
	Class<?> taskManagerMainClass = YarnTaskExecutorRunner.class;
	ContainerLaunchContext ctx;

	final Map<String, String> originalEnv = System.getenv();
	try {
		Map<String, String> systemEnv = new HashMap<>(originalEnv);
		systemEnv.put("HADOOP_TOKEN_FILE_LOCATION", credentialFile.getAbsolutePath());
		CommonTestUtils.setEnv(systemEnv);
		ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams,
			taskManagerConf, workingDirectory, taskManagerMainClass, LOG);
	} finally {
		CommonTestUtils.setEnv(originalEnv);
	}

	Credentials credentials = new Credentials();
	try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(ctx.getTokens().array()))) {
		credentials.readTokenStorageStream(dis);
	}
	Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens();
	boolean hasHdfsDelegationToken = false;
	boolean hasAmRmToken = false;
	for (Token<? extends TokenIdentifier> token : tokens) {
		if (token.getKind().equals(amRmTokenKind)) {
			hasAmRmToken = true;
		} else if (token.getKind().equals(hdfsDelegationTokenKind)) {
			hasHdfsDelegationToken = true;
		}
	}
	assertTrue(hasHdfsDelegationToken);
	assertFalse(hasAmRmToken);
}
 
Example 7
Source File: TestMRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testMRAppMasterCredentials() throws Exception {

  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);

  // Simulate credentials passed to AM via client->RM->NM
  Credentials credentials = new Credentials();
  byte[] identifier = "MyIdentifier".getBytes();
  byte[] password = "MyPassword".getBytes();
  Text kind = new Text("MyTokenKind");
  Text service = new Text("host:port");
  Token<? extends TokenIdentifier> myToken =
      new Token<TokenIdentifier>(identifier, password, kind, service);
  Text tokenAlias = new Text("myToken");
  credentials.addToken(tokenAlias, myToken);

  Text appTokenService = new Text("localhost:0");
  Token<AMRMTokenIdentifier> appToken =
      new Token<AMRMTokenIdentifier>(identifier, password,
          AMRMTokenIdentifier.KIND_NAME, appTokenService);
  credentials.addToken(appTokenService, appToken);
  
  Text keyAlias = new Text("mySecretKeyAlias");
  credentials.addSecretKey(keyAlias, "mySecretKey".getBytes());
  Token<? extends TokenIdentifier> storedToken =
      credentials.getToken(tokenAlias);

  JobConf conf = new JobConf();

  Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file");
  Map<String, String> newEnv = new HashMap<String, String>();
  newEnv.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, tokenFilePath
    .toUri().getPath());
  setNewEnvironmentHack(newEnv);
  credentials.writeTokenStorageFile(tokenFilePath, conf);

  ApplicationId appId = ApplicationId.newInstance(12345, 56);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 546);
  String userName = UserGroupInformation.getCurrentUser().getShortUserName();

  // Create staging dir, so MRAppMaster doesn't barf.
  File stagingDir =
      new File(MRApps.getStagingAreaDir(conf, userName).toString());
  stagingDir.mkdirs();

  // Set login-user to null as that is how real world MRApp starts with.
  // This is null is the reason why token-file is read by UGI.
  UserGroupInformation.setLoginUser(null);

  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
        System.currentTimeMillis(), false, true);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);

  // Now validate the task credentials
  Credentials appMasterCreds = appMaster.getCredentials();
  Assert.assertNotNull(appMasterCreds);
  Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys());
  Assert.assertEquals(1, appMasterCreds.numberOfTokens());

  // Validate the tokens - app token should not be present
  Token<? extends TokenIdentifier> usedToken =
      appMasterCreds.getToken(tokenAlias);
  Assert.assertNotNull(usedToken);
  Assert.assertEquals(storedToken, usedToken);

  // Validate the keys
  byte[] usedKey = appMasterCreds.getSecretKey(keyAlias);
  Assert.assertNotNull(usedKey);
  Assert.assertEquals("mySecretKey", new String(usedKey));

  // The credentials should also be added to conf so that OuputCommitter can
  // access it - app token should not be present
  Credentials confCredentials = conf.getCredentials();
  Assert.assertEquals(1, confCredentials.numberOfSecretKeys());
  Assert.assertEquals(1, confCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias));
  Assert.assertEquals("mySecretKey",
    new String(confCredentials.getSecretKey(keyAlias)));
  
  // Verify the AM's ugi - app token should be present
  Credentials ugiCredentials = appMaster.getUgi().getCredentials();
  Assert.assertEquals(1, ugiCredentials.numberOfSecretKeys());
  Assert.assertEquals(2, ugiCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, ugiCredentials.getToken(tokenAlias));
  Assert.assertEquals(appToken, ugiCredentials.getToken(appTokenService));
  Assert.assertEquals("mySecretKey",
    new String(ugiCredentials.getSecretKey(keyAlias)));


}
 
Example 8
Source File: TestGenericOptionsParser.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * testing -fileCache option
 * @throws IOException
 */
public void testTokenCacheOption() throws IOException {
  FileSystem localFs = FileSystem.getLocal(conf);
  
  File tmpFile = new File(testDir, "tokenCacheFile");
  if(tmpFile.exists()) {
    tmpFile.delete();
  }
  String[] args = new String[2];
  // pass a files option 
  args[0] = "-tokenCacheFile";
  args[1] = tmpFile.toURI().toString();
  
  // test non existing file
  Throwable th = null;
  try {
    new GenericOptionsParser(conf, args);
  } catch (Exception e) {
    th = e;
  }
  assertNotNull(th);
  assertTrue("FileNotFoundException is not thrown",
      th instanceof FileNotFoundException);
  
  // create file
  Path tmpPath = localFs.makeQualified(new Path(tmpFile.toString()));
  Token<?> token = new Token<AbstractDelegationTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(),
      new Text("token-kind"), new Text("token-service"));
  Credentials creds = new Credentials();
  creds.addToken(new Text("token-alias"), token);
  creds.writeTokenStorageFile(tmpPath, conf);

  new GenericOptionsParser(conf, args);
  String fileName = conf.get("mapreduce.job.credentials.binary");
  assertNotNull("files is null", fileName);
  assertEquals("files option does not match", tmpPath.toString(), fileName);
  
  Credentials ugiCreds =
      UserGroupInformation.getCurrentUser().getCredentials();
  assertEquals(1, ugiCreds.numberOfTokens());
  Token<?> ugiToken = ugiCreds.getToken(new Text("token-alias"));
  assertNotNull(ugiToken);
  assertEquals(token, ugiToken);
  
  localFs.delete(new Path(testDir.getAbsolutePath()), true);
}
 
Example 9
Source File: TestMRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMRAppMasterCredentials() throws Exception {

  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);

  // Simulate credentials passed to AM via client->RM->NM
  Credentials credentials = new Credentials();
  byte[] identifier = "MyIdentifier".getBytes();
  byte[] password = "MyPassword".getBytes();
  Text kind = new Text("MyTokenKind");
  Text service = new Text("host:port");
  Token<? extends TokenIdentifier> myToken =
      new Token<TokenIdentifier>(identifier, password, kind, service);
  Text tokenAlias = new Text("myToken");
  credentials.addToken(tokenAlias, myToken);

  Text appTokenService = new Text("localhost:0");
  Token<AMRMTokenIdentifier> appToken =
      new Token<AMRMTokenIdentifier>(identifier, password,
          AMRMTokenIdentifier.KIND_NAME, appTokenService);
  credentials.addToken(appTokenService, appToken);
  
  Text keyAlias = new Text("mySecretKeyAlias");
  credentials.addSecretKey(keyAlias, "mySecretKey".getBytes());
  Token<? extends TokenIdentifier> storedToken =
      credentials.getToken(tokenAlias);

  JobConf conf = new JobConf();

  Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file");
  Map<String, String> newEnv = new HashMap<String, String>();
  newEnv.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, tokenFilePath
    .toUri().getPath());
  setNewEnvironmentHack(newEnv);
  credentials.writeTokenStorageFile(tokenFilePath, conf);

  ApplicationId appId = ApplicationId.newInstance(12345, 56);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 546);
  String userName = UserGroupInformation.getCurrentUser().getShortUserName();

  // Create staging dir, so MRAppMaster doesn't barf.
  File stagingDir =
      new File(MRApps.getStagingAreaDir(conf, userName).toString());
  stagingDir.mkdirs();

  // Set login-user to null as that is how real world MRApp starts with.
  // This is null is the reason why token-file is read by UGI.
  UserGroupInformation.setLoginUser(null);

  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
        System.currentTimeMillis(), false, true);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);

  // Now validate the task credentials
  Credentials appMasterCreds = appMaster.getCredentials();
  Assert.assertNotNull(appMasterCreds);
  Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys());
  Assert.assertEquals(1, appMasterCreds.numberOfTokens());

  // Validate the tokens - app token should not be present
  Token<? extends TokenIdentifier> usedToken =
      appMasterCreds.getToken(tokenAlias);
  Assert.assertNotNull(usedToken);
  Assert.assertEquals(storedToken, usedToken);

  // Validate the keys
  byte[] usedKey = appMasterCreds.getSecretKey(keyAlias);
  Assert.assertNotNull(usedKey);
  Assert.assertEquals("mySecretKey", new String(usedKey));

  // The credentials should also be added to conf so that OuputCommitter can
  // access it - app token should not be present
  Credentials confCredentials = conf.getCredentials();
  Assert.assertEquals(1, confCredentials.numberOfSecretKeys());
  Assert.assertEquals(1, confCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias));
  Assert.assertEquals("mySecretKey",
    new String(confCredentials.getSecretKey(keyAlias)));
  
  // Verify the AM's ugi - app token should be present
  Credentials ugiCredentials = appMaster.getUgi().getCredentials();
  Assert.assertEquals(1, ugiCredentials.numberOfSecretKeys());
  Assert.assertEquals(2, ugiCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, ugiCredentials.getToken(tokenAlias));
  Assert.assertEquals(appToken, ugiCredentials.getToken(appTokenService));
  Assert.assertEquals("mySecretKey",
    new String(ugiCredentials.getSecretKey(keyAlias)));


}
 
Example 10
Source File: TestGenericOptionsParser.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * testing -fileCache option
 * @throws IOException
 */
public void testTokenCacheOption() throws IOException {
  FileSystem localFs = FileSystem.getLocal(conf);
  
  File tmpFile = new File(testDir, "tokenCacheFile");
  if(tmpFile.exists()) {
    tmpFile.delete();
  }
  String[] args = new String[2];
  // pass a files option 
  args[0] = "-tokenCacheFile";
  args[1] = tmpFile.toURI().toString();
  
  // test non existing file
  Throwable th = null;
  try {
    new GenericOptionsParser(conf, args);
  } catch (Exception e) {
    th = e;
  }
  assertNotNull(th);
  assertTrue("FileNotFoundException is not thrown",
      th instanceof FileNotFoundException);
  
  // create file
  Path tmpPath = localFs.makeQualified(new Path(tmpFile.toString()));
  Token<?> token = new Token<AbstractDelegationTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(),
      new Text("token-kind"), new Text("token-service"));
  Credentials creds = new Credentials();
  creds.addToken(new Text("token-alias"), token);
  creds.writeTokenStorageFile(tmpPath, conf);

  new GenericOptionsParser(conf, args);
  String fileName = conf.get("mapreduce.job.credentials.binary");
  assertNotNull("files is null", fileName);
  assertEquals("files option does not match", tmpPath.toString(), fileName);
  
  Credentials ugiCreds =
      UserGroupInformation.getCurrentUser().getCredentials();
  assertEquals(1, ugiCreds.numberOfTokens());
  Token<?> ugiToken = ugiCreds.getToken(new Text("token-alias"));
  assertNotNull(ugiToken);
  assertEquals(token, ugiToken);
  
  localFs.delete(new Path(testDir.getAbsolutePath()), true);
}
 
Example 11
Source File: UtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testCreateTaskExecutorCredentials() throws Exception {
	File root = temporaryFolder.getRoot();
	File home = new File(root, "home");
	boolean created = home.mkdir();
	assertTrue(created);

	Configuration flinkConf = new Configuration();
	YarnConfiguration yarnConf = new YarnConfiguration();

	Map<String, String> env = new HashMap<>();
	env.put(YarnConfigKeys.ENV_APP_ID, "foo");
	env.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, home.getAbsolutePath());
	env.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, "");
	env.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, "");
	env.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, "foo");
	env.put(YarnConfigKeys.FLINK_DIST_JAR, new YarnLocalResourceDescriptor(
		"flink.jar",
		new Path(root.toURI()),
		0,
		System.currentTimeMillis(),
		LocalResourceVisibility.APPLICATION).toString());
	env = Collections.unmodifiableMap(env);

	File credentialFile = temporaryFolder.newFile("container_tokens");
	final Text amRmTokenKind = AMRMTokenIdentifier.KIND_NAME;
	final Text hdfsDelegationTokenKind = new Text("HDFS_DELEGATION_TOKEN");
	final Text service = new Text("test-service");
	Credentials amCredentials = new Credentials();
	amCredentials.addToken(amRmTokenKind, new Token<>(new byte[4], new byte[4], amRmTokenKind, service));
	amCredentials.addToken(hdfsDelegationTokenKind, new Token<>(new byte[4], new byte[4],
		hdfsDelegationTokenKind, service));
	amCredentials.writeTokenStorageFile(new org.apache.hadoop.fs.Path(credentialFile.getAbsolutePath()), yarnConf);

	TaskExecutorProcessSpec spec = TaskExecutorProcessUtils
		.newProcessSpecBuilder(flinkConf)
		.withTotalProcessMemory(MemorySize.parse("1g"))
		.build();
	ContaineredTaskManagerParameters tmParams = new ContaineredTaskManagerParameters(spec, new HashMap<>(1));
	Configuration taskManagerConf = new Configuration();

	String workingDirectory = root.getAbsolutePath();
	Class<?> taskManagerMainClass = YarnTaskExecutorRunner.class;
	ContainerLaunchContext ctx;

	final Map<String, String> originalEnv = System.getenv();
	try {
		Map<String, String> systemEnv = new HashMap<>(originalEnv);
		systemEnv.put("HADOOP_TOKEN_FILE_LOCATION", credentialFile.getAbsolutePath());
		CommonTestUtils.setEnv(systemEnv);
		ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams,
			"", workingDirectory, taskManagerMainClass, LOG);
	} finally {
		CommonTestUtils.setEnv(originalEnv);
	}

	Credentials credentials = new Credentials();
	try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(ctx.getTokens().array()))) {
		credentials.readTokenStorageStream(dis);
	}
	Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens();
	boolean hasHdfsDelegationToken = false;
	boolean hasAmRmToken = false;
	for (Token<? extends TokenIdentifier> token : tokens) {
		if (token.getKind().equals(amRmTokenKind)) {
			hasAmRmToken = true;
		} else if (token.getKind().equals(hdfsDelegationTokenKind)) {
			hasHdfsDelegationToken = true;
		}
	}
	assertTrue(hasHdfsDelegationToken);
	assertFalse(hasAmRmToken);
}
 
Example 12
Source File: YarnHelixUtils.java    From incubator-gobblin with Apache License 2.0 3 votes vote down vote up
/**
 * Write a {@link Token} to a given file.
 *
 * @param token the token to write
 * @param tokenFilePath the token file path
 * @param configuration a {@link Configuration} object carrying Hadoop configuration properties
 * @throws IOException
 */
public static void writeTokenToFile(Token<? extends TokenIdentifier> token, Path tokenFilePath,
    Configuration configuration) throws IOException {
  Credentials credentials = new Credentials();
  credentials.addToken(token.getService(), token);
  credentials.writeTokenStorageFile(tokenFilePath, configuration);
}