org.apache.hadoop.yarn.api.records.Token Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.Token. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NMTokenSecretManagerInRM.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public NMToken createAndGetNMToken(String applicationSubmitter,
    ApplicationAttemptId appAttemptId, Container container) {
  try {
    this.readLock.lock();
    HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
    NMToken nmToken = null;
    if (nodeSet != null) {
      if (!nodeSet.contains(container.getNodeId())) {
        LOG.info("Sending NMToken for nodeId : " + container.getNodeId()
            + " for container : " + container.getId());
        Token token =
            createNMToken(container.getId().getApplicationAttemptId(),
              container.getNodeId(), applicationSubmitter);
        nmToken = NMToken.newInstance(container.getNodeId(), token);
        nodeSet.add(container.getNodeId());
      }
    }
    return nmToken;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #2
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #3
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #4
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private String testStartContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    boolean isExceptionExpected) {
  try {
    startContainer(rpc, nmToken, containerToken, nodeId,
        appAttemptId.toString());
    if (isExceptionExpected){
      fail("Exception was expected!!");        
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #5
Source File: AllocateResponse.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Private
@Unstable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
    List<ContainerResourceIncrease> increasedContainers,
    List<ContainerResourceDecrease> decreasedContainers) {
  AllocateResponse response =
      newInstance(responseId, completedContainers, allocatedContainers,
        updatedNodes, availResources, command, numClusterNodes, preempt,
        nmTokens, increasedContainers, decreasedContainers);
  response.setAMRMToken(amRMToken);
  return response;
}
 
Example #6
Source File: TestYarnApiClasses.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
* Test RenewDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/

@Test
public void testRenewDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  RenewDelegationTokenRequestPBImpl original = new RenewDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  RenewDelegationTokenRequestProto protoType = original.getProto();

  RenewDelegationTokenRequestPBImpl copy = new RenewDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}
 
Example #7
Source File: TestContainerResourceIncrease.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceIncreaseContext() {
  byte[] identifier = new byte[] { 1, 2, 3, 4 };
  Token token = Token.newInstance(identifier, "", "".getBytes(), "");
  ContainerId containerId = ContainerId
      .newContainerId(ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(1234, 3), 3), 7);
  Resource resource = Resource.newInstance(1023, 3);
  ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance(
      containerId, resource, token);

  // get proto and recover to ctx
  ContainerResourceIncreaseProto proto = 
      ((ContainerResourceIncreasePBImpl) ctx).getProto();
  ctx = new ContainerResourceIncreasePBImpl(proto);

  // check values
  Assert.assertEquals(ctx.getCapability(), resource);
  Assert.assertEquals(ctx.getContainerId(), containerId);
  Assert.assertTrue(Arrays.equals(ctx.getContainerToken().getIdentifier()
      .array(), identifier));
}
 
Example #8
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #9
Source File: NMTokenSecretManagerInRM.java    From big-c with Apache License 2.0 6 votes vote down vote up
public NMToken createAndGetNMToken(String applicationSubmitter,
    ApplicationAttemptId appAttemptId, Container container) {
  try {
    this.readLock.lock();
    HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
    NMToken nmToken = null;
    if (nodeSet != null) {
      if (!nodeSet.contains(container.getNodeId())) {
        LOG.info("Sending NMToken for nodeId : " + container.getNodeId()
            + " for container : " + container.getId());
        Token token =
            createNMToken(container.getId().getApplicationAttemptId(),
              container.getNodeId(), applicationSubmitter);
        nmToken = NMToken.newInstance(container.getNodeId(), token);
        nodeSet.add(container.getNodeId());
      }
    }
    return nmToken;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #10
Source File: TestYarnApiClasses.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
* Test CancelDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/
@Test
public void testCancelDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  CancelDelegationTokenRequestPBImpl original = new CancelDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  CancelDelegationTokenRequestProto protoType = original.getProto();

  CancelDelegationTokenRequestPBImpl copy = new CancelDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}
 
Example #11
Source File: GetDelegationTokenResponsePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (this.mrToken != null) {
    return this.mrToken;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.mrToken = convertFromProtoFormat(p.getToken());
  return this.mrToken;  
}
 
Example #12
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #13
Source File: NMClientAsyncImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ContainerEvent(ContainerId containerId, NodeId nodeId,
    Token containerToken, ContainerEventType type) {
  super(type);
  this.containerId = containerId;
  this.nodeId = nodeId;
  this.containerToken = containerToken;
}
 
Example #14
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager)
    throws IOException {
  return createContainerToken(cId, rmIdentifier, nodeId, user,
    containerTokenSecretManager, null);
}
 
Example #15
Source File: GetDelegationTokenResponse.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetDelegationTokenResponse newInstance(Token rmDTToken) {
  GetDelegationTokenResponse response =
      Records.newRecord(GetDelegationTokenResponse.class);
  response.setRMDelegationToken(rmDTToken);
  return response;
}
 
Example #16
Source File: ContainerStopRequest.java    From tez with Apache License 2.0 5 votes vote down vote up
public ContainerStopRequest(NodeId nodeId,
                            ContainerId containerId,
                            Token containerToken, String schedulerName, String taskCommName) {
  super(nodeId, containerId, containerToken);
  this.schedulerName = schedulerName;
  this.taskCommName = taskCommName;
}
 
Example #17
Source File: TestNMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Container mockContainer(int i) {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  ApplicationAttemptId attemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, i);
  nodeId = NodeId.newInstance("localhost", 0);
  // Create an empty record
  containerToken = recordFactory.newRecordInstance(Token.class);
  return Container.newInstance(containerId, nodeId, null, null, null,
    containerToken);
}
 
Example #18
Source File: ContainerLauncherOperationBase.java    From tez with Apache License 2.0 5 votes vote down vote up
public ContainerLauncherOperationBase(NodeId nodeId,
                                      ContainerId containerId,
                                      Token containerToken) {
  this.nodeId = nodeId;
  this.containerId = containerId;
  this.containerToken = containerToken;
}
 
Example #19
Source File: TestNMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Container mockContainer(int i) {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  ApplicationAttemptId attemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, i);
  nodeId = NodeId.newInstance("localhost", 0);
  // Create an empty record
  containerToken = recordFactory.newRecordInstance(Token.class);
  return Container.newInstance(containerId, nodeId, null, null, null,
    containerToken);
}
 
Example #20
Source File: GetDelegationTokenResponse.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetDelegationTokenResponse newInstance(Token rmDTToken) {
  GetDelegationTokenResponse response =
      Records.newRecord(GetDelegationTokenResponse.class);
  response.setRMDelegationToken(rmDTToken);
  return response;
}
 
Example #21
Source File: ApplicationReportPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setClientToAMToken(Token clientToAMToken) {
  maybeInitBuilder();
  if (clientToAMToken == null) 
    builder.clearClientToAmToken();
  this.clientToAMToken = clientToAMToken;
}
 
Example #22
Source File: NMTokenPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized Token getToken() {
  NMTokenProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.token = convertFromProtoFormat(p.getToken()); 
  return token;
}
 
Example #23
Source File: AMRMClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  // Preserve the token service sent by the RM when adding the token
  // to ensure we replace the previous token setup by the RM.
  // Afterwards we can update the service address for the RPC layer.
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
 
Example #24
Source File: NMCommunicatorEvent.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
    Token containerToken, NMCommunicatorEventType type) {
  super(type);
  this.containerId = containerId;
  this.nodeId = nodeId;
  this.containerToken = containerToken;
}
 
Example #25
Source File: ContainerPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public Token getContainerToken() {
  ContainerProtoOrBuilder p = viaProto ? proto : builder;
  if (this.containerToken != null) {
    return this.containerToken;
  }
  if (!p.hasContainerToken()) {
    return null;
  }
  this.containerToken = convertFromProtoFormat(p.getContainerToken());
  return this.containerToken;
}
 
Example #26
Source File: RenewDelegationTokenRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}
 
Example #27
Source File: StartContainerRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Public
@Stable
public static StartContainerRequest newInstance(
    ContainerLaunchContext context, Token container) {
  StartContainerRequest request =
      Records.newRecord(StartContainerRequest.class);
  request.setContainerLaunchContext(context);
  request.setContainerToken(container);
  return request;
}
 
Example #28
Source File: GetDelegationTokenResponsePBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (this.mrToken != null) {
    return this.mrToken;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.mrToken = convertFromProtoFormat(p.getToken());
  return this.mrToken;  
}
 
Example #29
Source File: TestApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #30
Source File: TestRPC.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}