Java Code Examples for org.apache.hadoop.security.token.Token#decodeIdentifier()

public DelegationTokenToRenew(Collection<ApplicationId> applicationIds,
    Token<?> token,
    Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
    String user) {
  this.token = token;
  this.user = user;
  if (token.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) {
    try {
      AbstractDelegationTokenIdentifier identifier =
          (AbstractDelegationTokenIdentifier) token.decodeIdentifier();
      maxDate = identifier.getMaxDate();
    } catch (IOException e) {
      throw new YarnRuntimeException(e);
    }
  }
  this.referringAppIds = Collections.synchronizedSet(
      new HashSet<ApplicationId>(applicationIds));
  this.conf = conf;
  this.expirationDate = expirationDate;
  this.timerTask = null;
  this.shouldCancelAtEnd = shouldCancelAtEnd;
}
 

/**
 * From a token, get the session token identifier.
 *
 * @param token token to process
 * @return the session token identifier
 * @throws IOException failure to validate/read data encoded in identifier.
 * @throws IllegalArgumentException if the token isn't an GCP session token
 */
public static DelegationTokenIdentifier extractIdentifier(
    final Token<? extends DelegationTokenIdentifier> token) throws IOException {
  checkArgument(token != null, "null token");
  DelegationTokenIdentifier identifier;
  // harden up decode beyond what Token does itself
  try {
    identifier = token.decodeIdentifier();
  } catch (RuntimeException e) {
    Throwable cause = e.getCause();
    if (cause != null) {
      // its a wrapping around class instantiation.
      throw new DelegationTokenIOException("Decoding GCS token " + cause, cause);
    }
    throw e;
  }
  if (identifier == null) {
    throw new DelegationTokenIOException("Failed to unmarshall token " + token);
  }
  return identifier;
}
 

private void verifyNewVersionToken(final Configuration conf, final CustomAM am,
    Token<ClientToAMTokenIdentifier> token, MockRM rm) throws IOException,
    InterruptedException {
  UserGroupInformation ugi;
  ugi = UserGroupInformation.createRemoteUser("me");
  
  Token<ClientToAMTokenIdentifier> newToken = 
      new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifierForTest(token.decodeIdentifier(), "message"),
          am.getClientToAMTokenSecretManager());
  newToken.setService(token.getService());
  
  ugi.addToken(newToken);

  ugi.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      CustomProtocol client =
          (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
            conf);
      client.ping();
      Assert.assertTrue(am.pinged);
      return null;
    }
  });
}
 

private void verifyNewVersionToken(final Configuration conf, final CustomAM am,
    Token<ClientToAMTokenIdentifier> token, MockRM rm) throws IOException,
    InterruptedException {
  UserGroupInformation ugi;
  ugi = UserGroupInformation.createRemoteUser("me");
  
  Token<ClientToAMTokenIdentifier> newToken = 
      new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifierForTest(token.decodeIdentifier(), "message"),
          am.getClientToAMTokenSecretManager());
  newToken.setService(token.getService());
  
  ugi.addToken(newToken);

  ugi.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      CustomProtocol client =
          (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
            conf);
      client.ping();
      Assert.assertTrue(am.pinged);
      return null;
    }
  });
}
 

public DelegationTokenToRenew(Collection<ApplicationId> applicationIds,
    Token<?> token,
    Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
    String user) {
  this.token = token;
  this.user = user;
  if (token.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) {
    try {
      AbstractDelegationTokenIdentifier identifier =
          (AbstractDelegationTokenIdentifier) token.decodeIdentifier();
      maxDate = identifier.getMaxDate();
    } catch (IOException e) {
      throw new YarnRuntimeException(e);
    }
  }
  this.referringAppIds = Collections.synchronizedSet(
      new HashSet<ApplicationId>(applicationIds));
  this.conf = conf;
  this.expirationDate = expirationDate;
  this.timerTask = null;
  this.shouldCancelAtEnd = shouldCancelAtEnd;
}
 

/**
 * Populate persisted password of AMRMToken back to AMRMTokenSecretManager.
 */
public void addPersistedPassword(Token<AMRMTokenIdentifier> token)
    throws IOException {
  this.writeLock.lock();
  try {
    AMRMTokenIdentifier identifier = token.decodeIdentifier();
    LOG.debug("Adding password for " + identifier.getApplicationAttemptId());
    appAttemptSet.add(identifier.getApplicationAttemptId());
  } finally {
    this.writeLock.unlock();
  }
}
 

/** Verifies that a configured delegation token binding is correctly loaded and employed */
@Test
public void testDelegationTokenBinding() throws IOException {
  URI initUri = new Path("gs://test/").toUri();
  Text expectedKind = TestTokenIdentifierImpl.KIND;

  GoogleHadoopFileSystem fs = new GoogleHadoopFileSystem();
  fs.initialize(initUri, loadConfig());

  // Request a delegation token
  Token<?> dt = fs.getDelegationToken(null);
  assertWithMessage("Expected a delegation token").that(dt).isNotNull();
  assertWithMessage("Unexpected delegation token service")
      .that(dt.getService().toString())
      .isEqualTo("gs://test");
  assertWithMessage("Unexpected delegation token kind")
      .that(dt.getKind())
      .isEqualTo(expectedKind);

  // Validate the associated identifier
  TokenIdentifier decoded = dt.decodeIdentifier();
  assertWithMessage("Failed to decode token identifier").that(decoded).isNotNull();
  assertWithMessage("Unexpected delegation token identifier type")
      .that(decoded)
      .isInstanceOf(TestTokenIdentifierImpl.class);

  DelegationTokenIdentifier identifier = (DelegationTokenIdentifier) decoded;
  assertWithMessage("Unexpected delegation token identifier kind")
      .that(identifier.getKind())
      .isEqualTo(expectedKind);
}
 

public ShadeSaslClientCallbackHandler(
    Token<? extends TokenIdentifier> token) throws IOException {
  TokenIdentifier id = token.decodeIdentifier();
  if (id == null) {
    // Something is wrong with the environment if we can't get our Identifier back out.
    throw new IllegalStateException("Could not extract Identifier from Token");
  }
  this.username = id.getUser().getUserName();
  this.password = Bytes.toString(token.getPassword()).toCharArray();
}
 

private Response createDelegationToken(DelegationToken tokenData,
    HttpServletRequest hsr, UserGroupInformation callerUGI)
    throws AuthorizationException, IOException, InterruptedException,
    Exception {

  final String renewer = tokenData.getRenewer();
  GetDelegationTokenResponse resp;
  try {
    resp =
        callerUGI
          .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
            @Override
            public GetDelegationTokenResponse run() throws IOException,
                YarnException {
              GetDelegationTokenRequest createReq =
                  GetDelegationTokenRequest.newInstance(renewer);
              return rm.getClientRMService().getDelegationToken(createReq);
            }
          });
  } catch (Exception e) {
    LOG.info("Create delegation token request failed", e);
    throw e;
  }

  Token<RMDelegationTokenIdentifier> tk =
      new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
        .getIdentifier().array(), resp.getRMDelegationToken().getPassword()
        .array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
        resp.getRMDelegationToken().getService()));
  RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
  long currentExpiration =
      rm.getRMContext().getRMDelegationTokenSecretManager()
        .getRenewDate(identifier);
  DelegationToken respToken =
      new DelegationToken(tk.encodeToUrlString(), renewer, identifier
        .getOwner().toString(), tk.getKind().toString(), currentExpiration,
        identifier.getMaxDate());
  return Response.status(Status.OK).entity(respToken).build();
}
 

private Response createDelegationToken(DelegationToken tokenData,
    HttpServletRequest hsr, UserGroupInformation callerUGI)
    throws AuthorizationException, IOException, InterruptedException,
    Exception {

  final String renewer = tokenData.getRenewer();
  GetDelegationTokenResponse resp;
  try {
    resp =
        callerUGI
          .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
            @Override
            public GetDelegationTokenResponse run() throws IOException,
                YarnException {
              GetDelegationTokenRequest createReq =
                  GetDelegationTokenRequest.newInstance(renewer);
              return rm.getClientRMService().getDelegationToken(createReq);
            }
          });
  } catch (Exception e) {
    LOG.info("Create delegation token request failed", e);
    throw e;
  }

  Token<RMDelegationTokenIdentifier> tk =
      new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
        .getIdentifier().array(), resp.getRMDelegationToken().getPassword()
        .array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
        resp.getRMDelegationToken().getService()));
  RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
  long currentExpiration =
      rm.getRMContext().getRMDelegationTokenSecretManager()
        .getRenewDate(identifier);
  DelegationToken respToken =
      new DelegationToken(tk.encodeToUrlString(), renewer, identifier
        .getOwner().toString(), tk.getKind().toString(), currentExpiration,
        identifier.getMaxDate());
  return Response.status(Status.OK).entity(respToken).build();
}
 

/**
 * Populate persisted password of AMRMToken back to AMRMTokenSecretManager.
 */
public void addPersistedPassword(Token<AMRMTokenIdentifier> token)
    throws IOException {
  this.writeLock.lock();
  try {
    AMRMTokenIdentifier identifier = token.decodeIdentifier();
    LOG.debug("Adding password for " + identifier.getApplicationAttemptId());
    appAttemptSet.add(identifier.getApplicationAttemptId());
  } finally {
    this.writeLock.unlock();
  }
}
 

private NMTokenIdentifier getNMTokenId(
    org.apache.hadoop.yarn.api.records.Token token) throws IOException {
  Token<NMTokenIdentifier> convertedToken =
      ConverterUtils.convertFromYarn(token, (Text) null);
  return convertedToken.decodeIdentifier();
}
 

private NMTokenIdentifier getNMTokenId(
    org.apache.hadoop.yarn.api.records.Token token) throws IOException {
  Token<NMTokenIdentifier> convertedToken =
      ConverterUtils.convertFromYarn(token, (Text) null);
  return convertedToken.decodeIdentifier();
}
 

@Test(timeout=10000)
public void testEditsForCancelOnTokenExpire() throws IOException,
InterruptedException {
  long renewInterval = 2000;
  Configuration conf = new Configuration();
  conf.setBoolean(
      DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
  conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, renewInterval);
  conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, renewInterval*2);

  Text renewer = new Text(UserGroupInformation.getCurrentUser().getUserName());
  FSImage fsImage = mock(FSImage.class);
  FSEditLog log = mock(FSEditLog.class);
  doReturn(log).when(fsImage).getEditLog();   
  FSNamesystem fsn = new FSNamesystem(conf, fsImage);
  
  DelegationTokenSecretManager dtsm = fsn.getDelegationTokenSecretManager();
  try {
    dtsm.startThreads();
    
    // get two tokens
    Token<DelegationTokenIdentifier> token1 = fsn.getDelegationToken(renewer);
    Token<DelegationTokenIdentifier> token2 = fsn.getDelegationToken(renewer);
    DelegationTokenIdentifier ident1 =
        token1.decodeIdentifier();
    DelegationTokenIdentifier ident2 =
        token2.decodeIdentifier();
    
    // verify we got the tokens
    verify(log, times(1)).logGetDelegationToken(eq(ident1), anyLong());
    verify(log, times(1)).logGetDelegationToken(eq(ident2), anyLong());
    
    // this is a little tricky because DTSM doesn't let us set scan interval
    // so need to periodically sleep, then stop/start threads to force scan
    
    // renew first token 1/2 to expire
    Thread.sleep(renewInterval/2);
    fsn.renewDelegationToken(token2);
    verify(log, times(1)).logRenewDelegationToken(eq(ident2), anyLong());
    // force scan and give it a little time to complete
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // no token has expired yet 
    verify(log, times(0)).logCancelDelegationToken(eq(ident1));
    verify(log, times(0)).logCancelDelegationToken(eq(ident2));
    
    // sleep past expiration of 1st non-renewed token
    Thread.sleep(renewInterval/2);
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // non-renewed token should have implicitly been cancelled
    verify(log, times(1)).logCancelDelegationToken(eq(ident1));
    verify(log, times(0)).logCancelDelegationToken(eq(ident2));
    
    // sleep past expiration of 2nd renewed token
    Thread.sleep(renewInterval/2);
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // both tokens should have been implicitly cancelled by now
    verify(log, times(1)).logCancelDelegationToken(eq(ident1));
    verify(log, times(1)).logCancelDelegationToken(eq(ident2));
  } finally {
    dtsm.stopThreads();
  }
}
 

@Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<RMDelegationTokenIdentifier, Long> rmDTState =
      rmState.getRMDTSecretManagerState().getTokenState();
  Set<DelegationKey> rmDTMasterKeyState =
      rmState.getRMDTSecretManagerState().getMasterKeyState();

  MockRM rm1 = new MyMockRM(conf, memStore);
  rm1.start();
  // on rm start, two master keys are created.
  // One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
  // the other is created on the first run of
  // tokenRemoverThread.rollMasterKey()

  RMDelegationTokenSecretManager dtSecretManager =
      rm1.getRMContext().getRMDelegationTokenSecretManager();
  // assert all master keys are saved
  Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
  Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>();
  expiringKeys.addAll(dtSecretManager.getAllMasterKeys());


  // request to generate a RMDelegationToken
  GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
  when(request.getRenewer()).thenReturn("renewer1");
  GetDelegationTokenResponse response =
      rm1.getClientRMService().getDelegationToken(request);
  org.apache.hadoop.yarn.api.records.Token delegationToken =
      response.getRMDelegationToken();
  Token<RMDelegationTokenIdentifier> token1 =
      ConverterUtils.convertFromYarn(delegationToken, (Text) null);
  RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();

  // For all keys that still remain in memory, we should have them stored
  // in state-store also.
  while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys
    .get() < 3) {
    ((TestRMDelegationTokenSecretManager) dtSecretManager)
      .checkCurrentKeyInStateStore(rmDTMasterKeyState);
    Thread.sleep(100);
  }

  // wait for token to expire and remove from state-store
  // rollMasterKey is called every 1 second.
  int count = 0;
  while (rmDTState.containsKey(dtId1) && count < 100) {
    Thread.sleep(100);
    count++;
  }
  rm1.stop();
}
 

@Test(timeout=10000)
public void testEditsForCancelOnTokenExpire() throws IOException,
InterruptedException {
  long renewInterval = 2000;
  Configuration conf = new Configuration();
  conf.setBoolean(
      DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
  conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, renewInterval);
  conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, renewInterval*2);

  Text renewer = new Text(UserGroupInformation.getCurrentUser().getUserName());
  FSImage fsImage = mock(FSImage.class);
  FSEditLog log = mock(FSEditLog.class);
  doReturn(log).when(fsImage).getEditLog();   
  FSNamesystem fsn = new FSNamesystem(conf, fsImage);
  
  DelegationTokenSecretManager dtsm = fsn.getDelegationTokenSecretManager();
  try {
    dtsm.startThreads();
    
    // get two tokens
    Token<DelegationTokenIdentifier> token1 = fsn.getDelegationToken(renewer);
    Token<DelegationTokenIdentifier> token2 = fsn.getDelegationToken(renewer);
    DelegationTokenIdentifier ident1 =
        token1.decodeIdentifier();
    DelegationTokenIdentifier ident2 =
        token2.decodeIdentifier();
    
    // verify we got the tokens
    verify(log, times(1)).logGetDelegationToken(eq(ident1), anyLong());
    verify(log, times(1)).logGetDelegationToken(eq(ident2), anyLong());
    
    // this is a little tricky because DTSM doesn't let us set scan interval
    // so need to periodically sleep, then stop/start threads to force scan
    
    // renew first token 1/2 to expire
    Thread.sleep(renewInterval/2);
    fsn.renewDelegationToken(token2);
    verify(log, times(1)).logRenewDelegationToken(eq(ident2), anyLong());
    // force scan and give it a little time to complete
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // no token has expired yet 
    verify(log, times(0)).logCancelDelegationToken(eq(ident1));
    verify(log, times(0)).logCancelDelegationToken(eq(ident2));
    
    // sleep past expiration of 1st non-renewed token
    Thread.sleep(renewInterval/2);
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // non-renewed token should have implicitly been cancelled
    verify(log, times(1)).logCancelDelegationToken(eq(ident1));
    verify(log, times(0)).logCancelDelegationToken(eq(ident2));
    
    // sleep past expiration of 2nd renewed token
    Thread.sleep(renewInterval/2);
    dtsm.stopThreads(); dtsm.startThreads();
    Thread.sleep(250);
    // both tokens should have been implicitly cancelled by now
    verify(log, times(1)).logCancelDelegationToken(eq(ident1));
    verify(log, times(1)).logCancelDelegationToken(eq(ident2));
  } finally {
    dtsm.stopThreads();
  }
}
 

@Test(timeout = 15000)
public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<RMDelegationTokenIdentifier, Long> rmDTState =
      rmState.getRMDTSecretManagerState().getTokenState();
  Set<DelegationKey> rmDTMasterKeyState =
      rmState.getRMDTSecretManagerState().getMasterKeyState();

  MockRM rm1 = new MyMockRM(conf, memStore);
  rm1.start();
  // on rm start, two master keys are created.
  // One is created at RMDTSecretMgr.startThreads.updateCurrentKey();
  // the other is created on the first run of
  // tokenRemoverThread.rollMasterKey()

  RMDelegationTokenSecretManager dtSecretManager =
      rm1.getRMContext().getRMDelegationTokenSecretManager();
  // assert all master keys are saved
  Assert.assertEquals(dtSecretManager.getAllMasterKeys(), rmDTMasterKeyState);
  Set<DelegationKey> expiringKeys = new HashSet<DelegationKey>();
  expiringKeys.addAll(dtSecretManager.getAllMasterKeys());


  // request to generate a RMDelegationToken
  GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
  when(request.getRenewer()).thenReturn("renewer1");
  GetDelegationTokenResponse response =
      rm1.getClientRMService().getDelegationToken(request);
  org.apache.hadoop.yarn.api.records.Token delegationToken =
      response.getRMDelegationToken();
  Token<RMDelegationTokenIdentifier> token1 =
      ConverterUtils.convertFromYarn(delegationToken, (Text) null);
  RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();

  // For all keys that still remain in memory, we should have them stored
  // in state-store also.
  while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys
    .get() < 3) {
    ((TestRMDelegationTokenSecretManager) dtSecretManager)
      .checkCurrentKeyInStateStore(rmDTMasterKeyState);
    Thread.sleep(100);
  }

  // wait for token to expire and remove from state-store
  // rollMasterKey is called every 1 second.
  int count = 0;
  while (rmDTState.containsKey(dtId1) && count < 100) {
    Thread.sleep(100);
    count++;
  }
  rm1.stop();
}
 

/**
 * Decode the token identifier. The subclass can customize the way to decode
 * the token identifier.
 * 
 * @param token the token where to extract the identifier
 * @return the delegation token identifier
 * @throws IOException
 */
public TokenIdent decodeTokenIdentifier(Token<TokenIdent> token) throws IOException {
  return token.decodeIdentifier();
}
 

/**
 * Decode the token identifier. The subclass can customize the way to decode
 * the token identifier.
 * 
 * @param token the token where to extract the identifier
 * @return the delegation token identifier
 * @throws IOException
 */
public TokenIdent decodeTokenIdentifier(Token<TokenIdent> token) throws IOException {
  return token.decodeIdentifier();
}