Java Code Examples for org.apache.hadoop.yarn.api.records.Token

The following examples show how to use org.apache.hadoop.yarn.api.records.Token. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: AllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Private
@Unstable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
    List<ContainerResourceIncrease> increasedContainers,
    List<ContainerResourceDecrease> decreasedContainers) {
  AllocateResponse response =
      newInstance(responseId, completedContainers, allocatedContainers,
        updatedNodes, availResources, command, numClusterNodes, preempt,
        nmTokens, increasedContainers, decreasedContainers);
  response.setAMRMToken(amRMToken);
  return response;
}
 
Example 2
Source Project: hadoop   Source File: TestYarnApiClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Test CancelDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/
@Test
public void testCancelDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  CancelDelegationTokenRequestPBImpl original = new CancelDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  CancelDelegationTokenRequestProto protoType = original.getProto();

  CancelDelegationTokenRequestPBImpl copy = new CancelDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}
 
Example 3
Source Project: big-c   Source File: TestContainerResourceIncrease.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceIncreaseContext() {
  byte[] identifier = new byte[] { 1, 2, 3, 4 };
  Token token = Token.newInstance(identifier, "", "".getBytes(), "");
  ContainerId containerId = ContainerId
      .newContainerId(ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(1234, 3), 3), 7);
  Resource resource = Resource.newInstance(1023, 3);
  ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance(
      containerId, resource, token);

  // get proto and recover to ctx
  ContainerResourceIncreaseProto proto = 
      ((ContainerResourceIncreasePBImpl) ctx).getProto();
  ctx = new ContainerResourceIncreasePBImpl(proto);

  // check values
  Assert.assertEquals(ctx.getCapability(), resource);
  Assert.assertEquals(ctx.getContainerId(), containerId);
  Assert.assertTrue(Arrays.equals(ctx.getContainerToken().getIdentifier()
      .array(), identifier));
}
 
Example 4
Source Project: big-c   Source File: NMTokenSecretManagerInRM.java    License: Apache License 2.0 6 votes vote down vote up
public NMToken createAndGetNMToken(String applicationSubmitter,
    ApplicationAttemptId appAttemptId, Container container) {
  try {
    this.readLock.lock();
    HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
    NMToken nmToken = null;
    if (nodeSet != null) {
      if (!nodeSet.contains(container.getNodeId())) {
        LOG.info("Sending NMToken for nodeId : " + container.getNodeId()
            + " for container : " + container.getId());
        Token token =
            createNMToken(container.getId().getApplicationAttemptId(),
              container.getNodeId(), applicationSubmitter);
        nmToken = NMToken.newInstance(container.getNodeId(), token);
        nodeSet.add(container.getNodeId());
      }
    }
    return nmToken;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 5
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example 6
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example 7
Source Project: big-c   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private String testStartContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    boolean isExceptionExpected) {
  try {
    startContainer(rpc, nmToken, containerToken, nodeId,
        appAttemptId.toString());
    if (isExceptionExpected){
      fail("Exception was expected!!");        
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example 8
Source Project: hadoop   Source File: TestYarnApiClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Test RenewDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/

@Test
public void testRenewDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  RenewDelegationTokenRequestPBImpl original = new RenewDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  RenewDelegationTokenRequestProto protoType = original.getProto();

  RenewDelegationTokenRequestPBImpl copy = new RenewDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}
 
Example 9
Source Project: hadoop   Source File: NMTokenSecretManagerInRM.java    License: Apache License 2.0 6 votes vote down vote up
public NMToken createAndGetNMToken(String applicationSubmitter,
    ApplicationAttemptId appAttemptId, Container container) {
  try {
    this.readLock.lock();
    HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
    NMToken nmToken = null;
    if (nodeSet != null) {
      if (!nodeSet.contains(container.getNodeId())) {
        LOG.info("Sending NMToken for nodeId : " + container.getNodeId()
            + " for container : " + container.getId());
        Token token =
            createNMToken(container.getId().getApplicationAttemptId(),
              container.getNodeId(), applicationSubmitter);
        nmToken = NMToken.newInstance(container.getNodeId(), token);
        nodeSet.add(container.getNodeId());
      }
    }
    return nmToken;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 10
Source Project: big-c   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example 11
Source Project: hadoop   Source File: CancelDelegationTokenRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  CancelDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}
 
Example 12
Source Project: big-c   Source File: TestRM.java    License: Apache License 2.0 5 votes vote down vote up
protected void allocateContainersAndValidateNMTokens(MockAM am,
    ArrayList<Container> containersReceived, int totalContainerRequested,
    HashMap<String, Token> nmTokens, MockNM nm) throws Exception,
    InterruptedException {
  ArrayList<ContainerId> releaseContainerList = new ArrayList<ContainerId>();
  AllocateResponse response;
  ArrayList<ResourceRequest> resourceRequest =
      new ArrayList<ResourceRequest>();      
  while (containersReceived.size() < totalContainerRequested) {
    nm.nodeHeartbeat(true);
    LOG.info("requesting containers..");
    response =
        am.allocate(resourceRequest, releaseContainerList);
    containersReceived.addAll(response.getAllocatedContainers());
    if (!response.getNMTokens().isEmpty()) {
      for (NMToken nmToken : response.getNMTokens()) {
        String nodeId = nmToken.getNodeId().toString();
        if (nmTokens.containsKey(nodeId)) {
          Assert.fail("Duplicate NMToken received for : " + nodeId);
        }
        nmTokens.put(nodeId, nmToken.getToken());
      }
    }
    LOG.info("Got " + containersReceived.size()
        + " containers. Waiting to get " + totalContainerRequested);
    Thread.sleep(WAIT_SLEEP_MS);
  }
}
 
Example 13
Source Project: samza   Source File: TestYarnClusterResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testErrorInStartContainerShouldUpdateState() {
  // create mocks
  final int samzaContainerId = 1;
  YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class);
  SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class);
  Config config = mock(Config.class);
  AMRMClientAsync asyncClient = mock(AMRMClientAsync.class);
  YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081);
  SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class);
  SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class);
  NMClientAsync asyncNMClient = mock(NMClientAsync.class);
  ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);

  // start the cluster manager
  YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient,
      callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);

  yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId),
      new YarnContainer(Container.newInstance(
          ContainerId.newContainerId(
              ApplicationAttemptId.newInstance(
                  ApplicationId.newInstance(10000L, 1), 1), 1),
          NodeId.newInstance("host1", 8088), "http://host1",
          Resource.newInstance(1024, 1), Priority.newInstance(1),
          Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service"))));

  yarnClusterResourceManager.start();
  assertEquals(1, yarnAppState.pendingProcessors.size());

  yarnClusterResourceManager.onStartContainerError(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(10000L, 1), 1), 1),
      new Exception());

  assertEquals(0, yarnAppState.pendingProcessors.size());
  verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class));
}
 
Example 14
Source Project: hadoop   Source File: RenewDelegationTokenRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}
 
Example 15
Source Project: hadoop   Source File: LocalContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
 
Example 16
Source Project: hadoop   Source File: StartContainerRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getContainerToken() {
  StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.containerToken != null) {
    return this.containerToken;
  }
  if (!p.hasContainerToken()) {
    return null;
  }
  this.containerToken = convertFromProtoFormat(p.getContainerToken());
  return this.containerToken;
}
 
Example 17
Source Project: big-c   Source File: LocalContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
 
Example 18
Source Project: hadoop   Source File: MRApp.java    License: Apache License 2.0 5 votes vote down vote up
public static Token newContainerToken(NodeId nodeId,
    byte[] password, ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      Token.newInstance(tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}
 
Example 19
Source Project: big-c   Source File: BaseNMTokenSecretManager.java    License: Apache License 2.0 5 votes vote down vote up
public static Token newInstance(byte[] password,
    NMTokenIdentifier identifier) {
  NodeId nodeId = identifier.getNodeId();
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  Token nmToken =
      Token.newInstance(identifier.getBytes(),
        NMTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return nmToken;
}
 
Example 20
Source Project: big-c   Source File: RenewDelegationTokenRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getDelegationToken() {
  RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}
 
Example 21
Source Project: hadoop   Source File: ContainerPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setContainerToken(Token containerToken) {
  maybeInitBuilder();
  if (containerToken == null) 
    builder.clearContainerToken();
  this.containerToken = containerToken;
}
 
Example 22
Source Project: hadoop   Source File: NMTokenPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setToken(Token token) {
  maybeInitBuilder();
  if (token == null) {
    builder.clearToken();
  }
  this.token = token;
}
 
Example 23
Source Project: big-c   Source File: MRApp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
 
Example 24
Source Project: big-c   Source File: GetDelegationTokenResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getRMDelegationToken() {
  GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (this.appToken != null) {
    return this.appToken;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.appToken = convertFromProtoFormat(p.getToken());
  return this.appToken;  
}
 
Example 25
Source Project: big-c   Source File: StartContainerRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getContainerToken() {
  StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.containerToken != null) {
    return this.containerToken;
  }
  if (!p.hasContainerToken()) {
    return null;
  }
  this.containerToken = convertFromProtoFormat(p.getContainerToken());
  return this.containerToken;
}
 
Example 26
Source Project: hadoop   Source File: ApplicationReportPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Token getAMRMToken() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (amRmToken != null) {
    return amRmToken;
  }
  if (!p.hasAmRmToken()) {
    return null;
  }
  amRmToken = convertFromProtoFormat(p.getAmRmToken());
  return amRmToken;
}
 
Example 27
Source Project: tez   Source File: ContainerLauncherEvent.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerLauncherEvent(ContainerId containerId, NodeId nodeId,
                              Token containerToken, ContainerLauncherEventType type,
                              int launcherId,
                              int schedulerId, int taskCommId) {
  super(type);
  this.containerId = containerId;
  this.nodeId = nodeId;
  this.containerToken = containerToken;
  this.launcherId = launcherId;
  this.schedulerId = schedulerId;
  this.taskCommId = taskCommId;
}
 
Example 28
Source Project: big-c   Source File: TestContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
protected Token createContainerToken(ContainerId cId, Priority priority,
    long createTime) throws InvalidToken {
  Resource r = BuilderUtils.newResource(1024, 1);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
        r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER,
        priority, createTime);
  Token containerToken =
      BuilderUtils.newContainerToken(
        context.getNodeId(),
        context.getContainerTokenSecretManager().retrievePassword(
          containerTokenIdentifier), containerTokenIdentifier);
  return containerToken;
}
 
Example 29
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
    getTimelineDelegationToken() throws IOException, YarnException {
      try {
        return timelineClient.getDelegationToken(timelineDTRenewer);
      } catch (Exception e ) {
        if (timelineServiceBestEffort) {
          LOG.warn("Failed to get delegation token from the timeline server: "
              + e.getMessage());
          return null;
        }
        throw e;
      }
}
 
Example 30
Source Project: hadoop   Source File: TestRPC.java    License: Apache License 2.0 5 votes vote down vote up
public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}