Java Code Examples for org.apache.hadoop.security.Credentials#getToken()

The following examples show how to use org.apache.hadoop.security.Credentials#getToken() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GcsDelegationTokens.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Look up a token from the credentials, verify it is of the correct kind.
 *
 * @param credentials credentials to look up.
 * @param service service name
 * @param kind token kind to look for
 * @return the token or null if no suitable token was found
 * @throws DelegationTokenIOException wrong token kind found
 */
@SuppressWarnings("unchecked") // safe by contract of lookupToken()
private static Token<DelegationTokenIdentifier> lookupToken(
    Credentials credentials, Text service, Text kind) throws DelegationTokenIOException {
  logger.atFine().log("Looking for token for service %s in credentials", service);
  Token<?> token = credentials.getToken(service);
  if (token != null) {
    Text tokenKind = token.getKind();
    logger.atFine().log("Found token of kind %s", tokenKind);
    if (kind.equals(tokenKind)) {
      // The OAuth implementation catches and logs here; this one throws the failure up.
      return (Token<DelegationTokenIdentifier>) token;
    }

    // There's a token for this service, but it's not the right DT kind
    throw DelegationTokenIOException.tokenMismatch(service, kind, tokenKind);
  }
  // A token for the service was not found
  logger.atFine().log("No token found for %s", service);
  return null;
}
 
Example 2
Source File: YARNRunner.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void addHistoryToken(Credentials ts) throws IOException, InterruptedException {
  /* check if we have a hsproxy, if not, no need */
  MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
  if (UserGroupInformation.isSecurityEnabled() && (hsProxy != null)) {
    /*
     * note that get delegation token was called. Again this is hack for oozie
     * to make sure we add history server delegation tokens to the credentials
     */
    RMDelegationTokenSelector tokenSelector = new RMDelegationTokenSelector();
    Text service = resMgrDelegate.getRMDelegationTokenService();
    if (tokenSelector.selectToken(service, ts.getAllTokens()) != null) {
      Text hsService = SecurityUtil.buildTokenService(hsProxy
          .getConnectAddress());
      if (ts.getToken(hsService) == null) {
        ts.addToken(hsService, getDelegationTokenFromHS(hsProxy));
      }
    }
  }
}
 
Example 3
Source File: FileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Recursively obtain the tokens for this FileSystem and all descended
 * FileSystems as determined by getChildFileSystems().
 * @param renewer the user allowed to renew the delegation tokens
 * @param credentials cache in which to add the new delegation tokens
 * @param tokens list in which to add acquired tokens
 * @throws IOException
 */
private void collectDelegationTokens(final String renewer,
                                     final Credentials credentials,
                                     final List<Token<?>> tokens)
                                         throws IOException {
  final String serviceName = getCanonicalServiceName();
  // Collect token of the this filesystem and then of its embedded children
  if (serviceName != null) { // fs has token, grab it
    final Text service = new Text(serviceName);
    Token<?> token = credentials.getToken(service);
    if (token == null) {
      token = getDelegationToken(renewer);
      if (token != null) {
        tokens.add(token);
        credentials.addToken(service, token);
      }
    }
  }
  // Now collect the tokens from the children
  final FileSystem[] children = getChildFileSystems();
  if (children != null) {
    for (final FileSystem fs : children) {
      fs.collectDelegationTokens(renewer, credentials, tokens);
    }
  }
}
 
Example 4
Source File: FileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Recursively obtain the tokens for this FileSystem and all descended
 * FileSystems as determined by getChildFileSystems().
 * @param renewer the user allowed to renew the delegation tokens
 * @param credentials cache in which to add the new delegation tokens
 * @param tokens list in which to add acquired tokens
 * @throws IOException
 */
private void collectDelegationTokens(final String renewer,
                                     final Credentials credentials,
                                     final List<Token<?>> tokens)
                                         throws IOException {
  final String serviceName = getCanonicalServiceName();
  // Collect token of the this filesystem and then of its embedded children
  if (serviceName != null) { // fs has token, grab it
    final Text service = new Text(serviceName);
    Token<?> token = credentials.getToken(service);
    if (token == null) {
      token = getDelegationToken(renewer);
      if (token != null) {
        tokens.add(token);
        credentials.addToken(service, token);
      }
    }
  }
  // Now collect the tokens from the children
  final FileSystem[] children = getChildFileSystems();
  if (children != null) {
    for (final FileSystem fs : children) {
      fs.collectDelegationTokens(renewer, credentials, tokens);
    }
  }
}
 
Example 5
Source File: YARNRunner.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void addHistoryToken(Credentials ts) throws IOException, InterruptedException {
  /* check if we have a hsproxy, if not, no need */
  MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
  if (UserGroupInformation.isSecurityEnabled() && (hsProxy != null)) {
    /*
     * note that get delegation token was called. Again this is hack for oozie
     * to make sure we add history server delegation tokens to the credentials
     */
    RMDelegationTokenSelector tokenSelector = new RMDelegationTokenSelector();
    Text service = resMgrDelegate.getRMDelegationTokenService();
    if (tokenSelector.selectToken(service, ts.getAllTokens()) != null) {
      Text hsService = SecurityUtil.buildTokenService(hsProxy
          .getConnectAddress());
      if (ts.getToken(hsService) == null) {
        ts.addToken(hsService, getDelegationTokenFromHS(hsProxy));
      }
    }
  }
}
 
Example 6
Source File: ResourceRequest.java    From sqoop-on-spark with Apache License 2.0 5 votes vote down vote up
public Token<?>[] addDelegationTokens(String strURL, String renewer,
                                      Credentials credentials) throws IOException {
  Token<?>[] tokens = null;
  Text dtService = getDelegationTokenService(strURL);
  Token<?> token = credentials.getToken(dtService);
  if (token == null) {
    URL url = new URL(strURL);
    DelegationTokenAuthenticatedURL authUrl =
            new DelegationTokenAuthenticatedURL(new ConnectionConfigurator() {
              @Override
              public HttpURLConnection configure(HttpURLConnection conn) throws IOException {
                return conn;
              }
            });
    try {
      token = authUrl.getDelegationToken(url, authToken, renewer);
      if (token != null) {
        credentials.addToken(token.getService(), token);
        tokens = new Token<?>[]{token};
      } else {
        throw new IOException("Got NULL as delegation token");
      }
    } catch (AuthenticationException ex) {
      throw new IOException(ex);
    }
  }
  return tokens;
}
 
Example 7
Source File: TokenCache.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * 
 * @return session token
 */
@SuppressWarnings("unchecked")
@InterfaceAudience.Private
public static Token<JobTokenIdentifier> getSessionToken(Credentials credentials) {
  Token<?> token = credentials.getToken(SESSION_TOKEN);
  if (token == null) {
    return null;
  }
  return (Token<JobTokenIdentifier>) token;
}
 
Example 8
Source File: TestTokenCache.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void checkToken(Credentials creds, Token<?> ... tokens) {
  assertEquals(tokens.length, creds.getAllTokens().size());
  for (Token<?> token : tokens) {
    Token<?> credsToken = creds.getToken(token.getService());
    assertTrue(credsToken != null);
    assertEquals(token, credsToken);
  }
}
 
Example 9
Source File: TokenCache.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * 
 * @return session token
 */
@SuppressWarnings("unchecked")
@InterfaceAudience.Private
public static Token<JobTokenIdentifier> getSessionToken(Credentials credentials) {
  Token<?> token = credentials.getToken(SESSION_TOKEN);
  if (token == null) {
    return null;
  }
  return (Token<JobTokenIdentifier>) token;
}
 
Example 10
Source File: TestTokenCache.java    From tez with Apache License 2.0 5 votes vote down vote up
private void checkTokens(Credentials creds, Credentials newCreds) {
  Assert.assertEquals(creds.getAllTokens().size(),
      newCreds.getAllTokens().size());
  for (Token<?> token : newCreds.getAllTokens()) {
    Token<?> credsToken = creds.getToken(token.getService());
    Assert.assertTrue(credsToken != null);
    Assert.assertEquals(token, credsToken);
  }
}
 
Example 11
Source File: TestTokenCache.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private void checkTokens(Credentials creds, Credentials newCreds) {
  Assert.assertEquals(creds.getAllTokens().size(),
      newCreds.getAllTokens().size());
  for (Token<?> token : newCreds.getAllTokens()) {
    Token<?> credsToken = creds.getToken(token.getService());
    Assert.assertTrue(credsToken != null);
    Assert.assertEquals(token, credsToken);
  }
}
 
Example 12
Source File: TestMRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMRAppMasterCredentials() throws Exception {

  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);

  // Simulate credentials passed to AM via client->RM->NM
  Credentials credentials = new Credentials();
  byte[] identifier = "MyIdentifier".getBytes();
  byte[] password = "MyPassword".getBytes();
  Text kind = new Text("MyTokenKind");
  Text service = new Text("host:port");
  Token<? extends TokenIdentifier> myToken =
      new Token<TokenIdentifier>(identifier, password, kind, service);
  Text tokenAlias = new Text("myToken");
  credentials.addToken(tokenAlias, myToken);

  Text appTokenService = new Text("localhost:0");
  Token<AMRMTokenIdentifier> appToken =
      new Token<AMRMTokenIdentifier>(identifier, password,
          AMRMTokenIdentifier.KIND_NAME, appTokenService);
  credentials.addToken(appTokenService, appToken);
  
  Text keyAlias = new Text("mySecretKeyAlias");
  credentials.addSecretKey(keyAlias, "mySecretKey".getBytes());
  Token<? extends TokenIdentifier> storedToken =
      credentials.getToken(tokenAlias);

  JobConf conf = new JobConf();

  Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file");
  Map<String, String> newEnv = new HashMap<String, String>();
  newEnv.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, tokenFilePath
    .toUri().getPath());
  setNewEnvironmentHack(newEnv);
  credentials.writeTokenStorageFile(tokenFilePath, conf);

  ApplicationId appId = ApplicationId.newInstance(12345, 56);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 546);
  String userName = UserGroupInformation.getCurrentUser().getShortUserName();

  // Create staging dir, so MRAppMaster doesn't barf.
  File stagingDir =
      new File(MRApps.getStagingAreaDir(conf, userName).toString());
  stagingDir.mkdirs();

  // Set login-user to null as that is how real world MRApp starts with.
  // This is null is the reason why token-file is read by UGI.
  UserGroupInformation.setLoginUser(null);

  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
        System.currentTimeMillis(), false, true);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);

  // Now validate the task credentials
  Credentials appMasterCreds = appMaster.getCredentials();
  Assert.assertNotNull(appMasterCreds);
  Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys());
  Assert.assertEquals(1, appMasterCreds.numberOfTokens());

  // Validate the tokens - app token should not be present
  Token<? extends TokenIdentifier> usedToken =
      appMasterCreds.getToken(tokenAlias);
  Assert.assertNotNull(usedToken);
  Assert.assertEquals(storedToken, usedToken);

  // Validate the keys
  byte[] usedKey = appMasterCreds.getSecretKey(keyAlias);
  Assert.assertNotNull(usedKey);
  Assert.assertEquals("mySecretKey", new String(usedKey));

  // The credentials should also be added to conf so that OuputCommitter can
  // access it - app token should not be present
  Credentials confCredentials = conf.getCredentials();
  Assert.assertEquals(1, confCredentials.numberOfSecretKeys());
  Assert.assertEquals(1, confCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias));
  Assert.assertEquals("mySecretKey",
    new String(confCredentials.getSecretKey(keyAlias)));
  
  // Verify the AM's ugi - app token should be present
  Credentials ugiCredentials = appMaster.getUgi().getCredentials();
  Assert.assertEquals(1, ugiCredentials.numberOfSecretKeys());
  Assert.assertEquals(2, ugiCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, ugiCredentials.getToken(tokenAlias));
  Assert.assertEquals(appToken, ugiCredentials.getToken(appTokenService));
  Assert.assertEquals("mySecretKey",
    new String(ugiCredentials.getSecretKey(keyAlias)));


}
 
Example 13
Source File: TestGenericOptionsParser.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * testing -fileCache option
 * @throws IOException
 */
public void testTokenCacheOption() throws IOException {
  FileSystem localFs = FileSystem.getLocal(conf);
  
  File tmpFile = new File(testDir, "tokenCacheFile");
  if(tmpFile.exists()) {
    tmpFile.delete();
  }
  String[] args = new String[2];
  // pass a files option 
  args[0] = "-tokenCacheFile";
  args[1] = tmpFile.toURI().toString();
  
  // test non existing file
  Throwable th = null;
  try {
    new GenericOptionsParser(conf, args);
  } catch (Exception e) {
    th = e;
  }
  assertNotNull(th);
  assertTrue("FileNotFoundException is not thrown",
      th instanceof FileNotFoundException);
  
  // create file
  Path tmpPath = localFs.makeQualified(new Path(tmpFile.toString()));
  Token<?> token = new Token<AbstractDelegationTokenIdentifier>(
      "identifier".getBytes(), "password".getBytes(),
      new Text("token-kind"), new Text("token-service"));
  Credentials creds = new Credentials();
  creds.addToken(new Text("token-alias"), token);
  creds.writeTokenStorageFile(tmpPath, conf);

  new GenericOptionsParser(conf, args);
  String fileName = conf.get("mapreduce.job.credentials.binary");
  assertNotNull("files is null", fileName);
  assertEquals("files option does not match", tmpPath.toString(), fileName);
  
  Credentials ugiCreds =
      UserGroupInformation.getCurrentUser().getCredentials();
  assertEquals(1, ugiCreds.numberOfTokens());
  Token<?> ugiToken = ugiCreds.getToken(new Text("token-alias"));
  assertNotNull(ugiToken);
  assertEquals(token, ugiToken);
  
  localFs.delete(new Path(testDir.getAbsolutePath()), true);
}
 
Example 14
Source File: TestDAGAppMaster.java    From tez with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
private void testDagCredentials(boolean doMerge) throws IOException {
  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge);
  conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
  conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString());
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);

  // create some sample AM credentials
  Credentials amCreds = new Credentials();
  JobTokenSecretManager jtsm = new JobTokenSecretManager();
  JobTokenIdentifier identifier = new JobTokenIdentifier(
      new Text(appId.toString()));
  Token<JobTokenIdentifier> sessionToken =
      new Token<JobTokenIdentifier>(identifier, jtsm);
  sessionToken.setService(identifier.getJobId());
  TokenCache.setSessionToken(sessionToken, amCreds);
  TestTokenSecretManager ttsm = new TestTokenSecretManager();
  Text tokenAlias1 = new Text("alias1");
  Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("amtoken1")), ttsm);
  amCreds.addToken(tokenAlias1, amToken1);
  Text tokenAlias2 = new Text("alias2");
  Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("amtoken2")), ttsm);
  amCreds.addToken(tokenAlias2, amToken2);

  FileSystem fs = FileSystem.getLocal(conf);
  FSDataOutputStream sessionJarsPBOutStream =
      TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(),
          TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
  DAGProtos.PlanLocalResourcesProto.getDefaultInstance()
      .writeDelimitedTo(sessionJarsPBOutStream);
  sessionJarsPBOutStream.close();
  DAGAppMaster am = new DAGAppMaster(attemptId,
      ContainerId.newInstance(attemptId, 1),
      "127.0.0.1", 0, 0, new SystemClock(), 1, true,
      TEST_DIR.toString(), new String[] {TEST_DIR.toString()},
      new String[] {TEST_DIR.toString()},
      new TezApiVersionInfo().getVersion(), amCreds,
      "someuser", null);
  am.init(conf);
  am.start();

  // create some sample DAG credentials
  Credentials dagCreds = new Credentials();
  Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("dagtoken1")), ttsm);
  dagCreds.addToken(tokenAlias2, dagToken1);
  Text tokenAlias3 = new Text("alias3");
  Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>(
      new TestTokenIdentifier(new Text("dagtoken2")), ttsm);
  dagCreds.addToken(tokenAlias3, dagToken2);

  TezDAGID dagId = TezDAGID.getInstance(appId, 1);
  DAGPlan dagPlan = DAGPlan.newBuilder()
      .setName("somedag")
      .setCredentialsBinary(
          DagTypeConverters.convertCredentialsToProto(dagCreds))
      .build();
  DAGImpl dag = am.createDAG(dagPlan, dagId);
  Credentials fetchedDagCreds = dag.getCredentials();
  am.stop();

  Token<? extends TokenIdentifier> fetchedToken1 =
      fetchedDagCreds.getToken(tokenAlias1);
  if (doMerge) {
    assertNotNull("AM creds missing from DAG creds", fetchedToken1);
    compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1));
  } else {
    assertNull("AM creds leaked to DAG creds", fetchedToken1);
  }
  compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2));
  compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3));
}
 
Example 15
Source File: TokenCache.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * 
 * @return job token
 */
@SuppressWarnings("unchecked")
@InterfaceAudience.Private
public static Token<JobTokenIdentifier> getJobToken(Credentials credentials) {
  return (Token<JobTokenIdentifier>) credentials.getToken(JOB_TOKEN);
}
 
Example 16
Source File: TestTaskAttemptContainerRequest.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testAttemptContainerRequest() throws Exception {
  final Text SECRET_KEY_ALIAS = new Text("secretkeyalias");
  final byte[] SECRET_KEY = ("secretkey").getBytes();
  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(1);
  acls.put(ApplicationAccessType.VIEW_APP, "otheruser");
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  Path jobFile = mock(Path.class);

  EventHandler eventHandler = mock(EventHandler.class);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");

  // setup UGI for security so tokens and keys are preserved
  jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(jobConf);

  Credentials credentials = new Credentials();
  credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY);
  Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
      ("tokenid").getBytes(), ("tokenpw").getBytes(),
      new Text("tokenkind"), new Text("tokenservice"));

  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          mock(TaskSplitMetaInfo.class), jobConf, taListener,
          jobToken, credentials,
          new SystemClock(), null);

  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString());

  ContainerLaunchContext launchCtx =
      TaskAttemptImpl.createContainerLaunchContext(acls,
          jobConf, jobToken, taImpl.createRemoteTask(),
          TypeConverter.fromYarn(jobId),
          mock(WrappedJvmID.class), taListener,
          credentials);

  Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs());
  Credentials launchCredentials = new Credentials();

  DataInputByteBuffer dibb = new DataInputByteBuffer();
  dibb.reset(launchCtx.getTokens());
  launchCredentials.readTokenStorageStream(dibb);

  // verify all tokens specified for the task attempt are in the launch context
  for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
    Token<? extends TokenIdentifier> launchToken =
        launchCredentials.getToken(token.getService());
    Assert.assertNotNull("Token " + token.getService() + " is missing",
        launchToken);
    Assert.assertEquals("Token " + token.getService() + " mismatch",
        token, launchToken);
  }

  // verify the secret key is in the launch context
  Assert.assertNotNull("Secret key missing",
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS));
  Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY,
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS)));
}
 
Example 17
Source File: TestTaskAttemptContainerRequest.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testAttemptContainerRequest() throws Exception {
  final Text SECRET_KEY_ALIAS = new Text("secretkeyalias");
  final byte[] SECRET_KEY = ("secretkey").getBytes();
  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(1);
  acls.put(ApplicationAccessType.VIEW_APP, "otheruser");
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  Path jobFile = mock(Path.class);

  EventHandler eventHandler = mock(EventHandler.class);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");

  // setup UGI for security so tokens and keys are preserved
  jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(jobConf);

  Credentials credentials = new Credentials();
  credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY);
  Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
      ("tokenid").getBytes(), ("tokenpw").getBytes(),
      new Text("tokenkind"), new Text("tokenservice"));

  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          mock(TaskSplitMetaInfo.class), jobConf, taListener,
          jobToken, credentials,
          new SystemClock(), null);

  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString());

  ContainerLaunchContext launchCtx =
      TaskAttemptImpl.createContainerLaunchContext(acls,
          jobConf, jobToken, taImpl.createRemoteTask(),
          TypeConverter.fromYarn(jobId),
          mock(WrappedJvmID.class), taListener,
          credentials);

  Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs());
  Credentials launchCredentials = new Credentials();

  DataInputByteBuffer dibb = new DataInputByteBuffer();
  dibb.reset(launchCtx.getTokens());
  launchCredentials.readTokenStorageStream(dibb);

  // verify all tokens specified for the task attempt are in the launch context
  for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) {
    Token<? extends TokenIdentifier> launchToken =
        launchCredentials.getToken(token.getService());
    Assert.assertNotNull("Token " + token.getService() + " is missing",
        launchToken);
    Assert.assertEquals("Token " + token.getService() + " mismatch",
        token, launchToken);
  }

  // verify the secret key is in the launch context
  Assert.assertNotNull("Secret key missing",
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS));
  Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY,
      launchCredentials.getSecretKey(SECRET_KEY_ALIAS)));
}
 
Example 18
Source File: DelegationTokenAuthenticatedURL.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Returns an authenticated {@link HttpURLConnection}. If the Delegation
 * Token is present, it will be used taking precedence over the configured
 * <code>Authenticator</code>. If the <code>doAs</code> parameter is not NULL,
 * the request will be done on behalf of the specified <code>doAs</code> user.
 *
 * @param url the URL to connect to. Only HTTP/S URLs are supported.
 * @param token the authentication token being used for the user.
 * @param doAs user to do the the request on behalf of, if NULL the request is
 * as self.
 * @return an authenticated {@link HttpURLConnection}.
 * @throws IOException if an IO error occurred.
 * @throws AuthenticationException if an authentication exception occurred.
 */
@SuppressWarnings("unchecked")
public HttpURLConnection openConnection(URL url, Token token, String doAs)
    throws IOException, AuthenticationException {
  Preconditions.checkNotNull(url, "url");
  Preconditions.checkNotNull(token, "token");
  Map<String, String> extraParams = new HashMap<String, String>();
  org.apache.hadoop.security.token.Token<? extends TokenIdentifier> dToken
      = null;
  // if we have valid auth token, it takes precedence over a delegation token
  // and we don't even look for one.
  if (!token.isSet()) {
    // delegation token
    Credentials creds = UserGroupInformation.getCurrentUser().
        getCredentials();
    if (!creds.getAllTokens().isEmpty()) {
      InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(),
          url.getPort());
      Text service = SecurityUtil.buildTokenService(serviceAddr);
      dToken = creds.getToken(service);
      if (dToken != null) {
        if (useQueryStringForDelegationToken()) {
          // delegation token will go in the query string, injecting it
          extraParams.put(
              KerberosDelegationTokenAuthenticator.DELEGATION_PARAM,
              dToken.encodeToUrlString());
        } else {
          // delegation token will go as request header, setting it in the
          // auth-token to ensure no authentication handshake is triggered
          // (if we have a delegation token, we are authenticated)
          // the delegation token header is injected in the connection request
          // at the end of this method.
          token.delegationToken = (org.apache.hadoop.security.token.Token
              <AbstractDelegationTokenIdentifier>) dToken;
        }
      }
    }
  }

  // proxyuser
  if (doAs != null) {
    extraParams.put(DO_AS, URLEncoder.encode(doAs, "UTF-8"));
  }

  url = augmentURL(url, extraParams);
  HttpURLConnection conn = super.openConnection(url, token);
  if (!token.isSet() && !useQueryStringForDelegationToken() && dToken != null) {
    // injecting the delegation token header in the connection request
    conn.setRequestProperty(
        DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER,
        dToken.encodeToUrlString());
  }
  return conn;
}
 
Example 19
Source File: TokenCache.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * 
 * @return job token
 */
@SuppressWarnings("unchecked")
@InterfaceAudience.Private
public static Token<JobTokenIdentifier> getJobToken(Credentials credentials) {
  return (Token<JobTokenIdentifier>) credentials.getToken(JOB_TOKEN);
}
 
Example 20
Source File: TestMRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testMRAppMasterCredentials() throws Exception {

  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);

  // Simulate credentials passed to AM via client->RM->NM
  Credentials credentials = new Credentials();
  byte[] identifier = "MyIdentifier".getBytes();
  byte[] password = "MyPassword".getBytes();
  Text kind = new Text("MyTokenKind");
  Text service = new Text("host:port");
  Token<? extends TokenIdentifier> myToken =
      new Token<TokenIdentifier>(identifier, password, kind, service);
  Text tokenAlias = new Text("myToken");
  credentials.addToken(tokenAlias, myToken);

  Text appTokenService = new Text("localhost:0");
  Token<AMRMTokenIdentifier> appToken =
      new Token<AMRMTokenIdentifier>(identifier, password,
          AMRMTokenIdentifier.KIND_NAME, appTokenService);
  credentials.addToken(appTokenService, appToken);
  
  Text keyAlias = new Text("mySecretKeyAlias");
  credentials.addSecretKey(keyAlias, "mySecretKey".getBytes());
  Token<? extends TokenIdentifier> storedToken =
      credentials.getToken(tokenAlias);

  JobConf conf = new JobConf();

  Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file");
  Map<String, String> newEnv = new HashMap<String, String>();
  newEnv.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, tokenFilePath
    .toUri().getPath());
  setNewEnvironmentHack(newEnv);
  credentials.writeTokenStorageFile(tokenFilePath, conf);

  ApplicationId appId = ApplicationId.newInstance(12345, 56);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 546);
  String userName = UserGroupInformation.getCurrentUser().getShortUserName();

  // Create staging dir, so MRAppMaster doesn't barf.
  File stagingDir =
      new File(MRApps.getStagingAreaDir(conf, userName).toString());
  stagingDir.mkdirs();

  // Set login-user to null as that is how real world MRApp starts with.
  // This is null is the reason why token-file is read by UGI.
  UserGroupInformation.setLoginUser(null);

  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
        System.currentTimeMillis(), false, true);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);

  // Now validate the task credentials
  Credentials appMasterCreds = appMaster.getCredentials();
  Assert.assertNotNull(appMasterCreds);
  Assert.assertEquals(1, appMasterCreds.numberOfSecretKeys());
  Assert.assertEquals(1, appMasterCreds.numberOfTokens());

  // Validate the tokens - app token should not be present
  Token<? extends TokenIdentifier> usedToken =
      appMasterCreds.getToken(tokenAlias);
  Assert.assertNotNull(usedToken);
  Assert.assertEquals(storedToken, usedToken);

  // Validate the keys
  byte[] usedKey = appMasterCreds.getSecretKey(keyAlias);
  Assert.assertNotNull(usedKey);
  Assert.assertEquals("mySecretKey", new String(usedKey));

  // The credentials should also be added to conf so that OuputCommitter can
  // access it - app token should not be present
  Credentials confCredentials = conf.getCredentials();
  Assert.assertEquals(1, confCredentials.numberOfSecretKeys());
  Assert.assertEquals(1, confCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, confCredentials.getToken(tokenAlias));
  Assert.assertEquals("mySecretKey",
    new String(confCredentials.getSecretKey(keyAlias)));
  
  // Verify the AM's ugi - app token should be present
  Credentials ugiCredentials = appMaster.getUgi().getCredentials();
  Assert.assertEquals(1, ugiCredentials.numberOfSecretKeys());
  Assert.assertEquals(2, ugiCredentials.numberOfTokens());
  Assert.assertEquals(storedToken, ugiCredentials.getToken(tokenAlias));
  Assert.assertEquals(appToken, ugiCredentials.getToken(appTokenService));
  Assert.assertEquals("mySecretKey",
    new String(ugiCredentials.getSecretKey(keyAlias)));


}