Java Code Examples for org.apache.hadoop.yarn.security.ContainerTokenIdentifier#getContainerID()

The following examples show how to use org.apache.hadoop.yarn.security.ContainerTokenIdentifier#getContainerID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NMContainerTokenSecretManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Container start has gone through. We need to store the containerId in order
 * to block future container start requests with same container token. This
 * container token needs to be saved till its container token expires.
 */
public synchronized void startContainerSuccessful(
    ContainerTokenIdentifier tokenId) {

  removeAnyContainerTokenIfExpired();
  
  ContainerId containerId = tokenId.getContainerID();
  Long expTime = tokenId.getExpiryTimeStamp();
  // We might have multiple containers with same expiration time.
  if (!recentlyStartedContainerTracker.containsKey(expTime)) {
    recentlyStartedContainerTracker
      .put(expTime, new ArrayList<ContainerId>());
  }
  recentlyStartedContainerTracker.get(expTime).add(containerId);
  try {
    stateStore.storeContainerToken(containerId, expTime);
  } catch (IOException e) {
    LOG.error("Unable to store token for container " + containerId, e);
  }
}
Example 2
Source File: ContainerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  stateMachine = stateMachineFactory.make(this);
}
Example 3
Source File: NMContainerTokenSecretManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Container start has gone through. We need to store the containerId in order
 * to block future container start requests with same container token. This
 * container token needs to be saved till its container token expires.
 */
public synchronized void startContainerSuccessful(
    ContainerTokenIdentifier tokenId) {

  removeAnyContainerTokenIfExpired();
  
  ContainerId containerId = tokenId.getContainerID();
  Long expTime = tokenId.getExpiryTimeStamp();
  // We might have multiple containers with same expiration time.
  if (!recentlyStartedContainerTracker.containsKey(expTime)) {
    recentlyStartedContainerTracker
      .put(expTime, new ArrayList<ContainerId>());
  }
  recentlyStartedContainerTracker.get(expTime).add(containerId);
  try {
    stateStore.storeContainerToken(containerId, expTime);
  } catch (IOException e) {
    LOG.error("Unable to store token for container " + containerId, e);
  }
}
Example 4
Source File: ContainerImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,Set<Integer> cpuCores) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.currentResource = resource;
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();
  this.cpuCores  = cpuCores;
  this.context = context;

  stateMachine = stateMachineFactory.make(this);
}
Example 5
Source File: ContainerTokenIdentifierForTest.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public ContainerTokenIdentifierForTest(ContainerTokenIdentifier identifier,
    String message) {
  ContainerTokenIdentifierForTestProto.Builder builder =
      ContainerTokenIdentifierForTestProto.newBuilder();
  ContainerIdPBImpl containerID = 
      (ContainerIdPBImpl)identifier.getContainerID();
  if (containerID != null) {
    builder.setContainerId(containerID.getProto());
  }
  builder.setNmHostAddr(identifier.getNmHostAddress());
  builder.setAppSubmitter(identifier.getApplicationSubmitter());
  
  ResourcePBImpl resource = (ResourcePBImpl)identifier.getResource();
  if (resource != null) {
    builder.setResource(resource.getProto());
  }
  
  builder.setExpiryTimeStamp(identifier.getExpiryTimeStamp());
  builder.setMasterKeyId(identifier.getMasterKeyId());
  builder.setRmIdentifier(identifier.getRMIdentifier());
  
  PriorityPBImpl priority = (PriorityPBImpl)identifier.getPriority();
  if (priority != null) {
    builder.setPriority(priority.getProto());
  }
  
  builder.setCreationTime(identifier.getCreationTime());
  builder.setMessage(message);
  
  LogAggregationContextPBImpl logAggregationContext = 
      (LogAggregationContextPBImpl)identifier.getLogAggregationContext();
  
  if (logAggregationContext != null) {
    builder.setLogAggregationContext(logAggregationContext.getProto());
  }
  
  proto = builder.build();
}
Example 6
Source File: ContainerManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());

  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled());
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
Example 7
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
Example 8
Source File: ContainerTokenIdentifierForTest.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public ContainerTokenIdentifierForTest(ContainerTokenIdentifier identifier,
    String message) {
  ContainerTokenIdentifierForTestProto.Builder builder =
      ContainerTokenIdentifierForTestProto.newBuilder();
  ContainerIdPBImpl containerID = 
      (ContainerIdPBImpl)identifier.getContainerID();
  if (containerID != null) {
    builder.setContainerId(containerID.getProto());
  }
  builder.setNmHostAddr(identifier.getNmHostAddress());
  builder.setAppSubmitter(identifier.getApplicationSubmitter());
  
  ResourcePBImpl resource = (ResourcePBImpl)identifier.getResource();
  if (resource != null) {
    builder.setResource(resource.getProto());
  }
  
  builder.setExpiryTimeStamp(identifier.getExpiryTimeStamp());
  builder.setMasterKeyId(identifier.getMasterKeyId());
  builder.setRmIdentifier(identifier.getRMIdentifier());
  
  PriorityPBImpl priority = (PriorityPBImpl)identifier.getPriority();
  if (priority != null) {
    builder.setPriority(priority.getProto());
  }
  
  builder.setCreationTime(identifier.getCreationTime());
  builder.setMessage(message);
  
  LogAggregationContextPBImpl logAggregationContext = 
      (LogAggregationContextPBImpl)identifier.getLogAggregationContext();
  
  if (logAggregationContext != null) {
    builder.setLogAggregationContext(logAggregationContext.getProto());
  }
  
  proto = builder.build();
}
Example 9
Source File: ContainerManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());
  
  Set<Integer> cores= this.context.getCoresManager().allocateCores(containerId, 
  		                          token.getResource().getVirtualCores());
 
  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(this.context,getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled(),cores);
    
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
Example 10
Source File: TestApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
Example 11
Source File: ContainerManagerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * @param containerTokenIdentifier
 *          of the container to be started
 * @throws YarnException
 */
@Private
@VisibleForTesting
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
    ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {

  ContainerId containerId = containerTokenIdentifier.getContainerID();
  String containerIDStr = containerId.toString();
  boolean unauthorized = false;
  StringBuilder messageBuilder =
      new StringBuilder("Unauthorized request to start container. ");
  if (!nmTokenIdentifier.getApplicationAttemptId().getApplicationId().
      equals(containerId.getApplicationAttemptId().getApplicationId())) {
    unauthorized = true;
    messageBuilder.append("\nNMToken for application attempt : ")
      .append(nmTokenIdentifier.getApplicationAttemptId())
      .append(" was used for starting container with container token")
      .append(" issued for application attempt : ")
      .append(containerId.getApplicationAttemptId());
  } else if (!this.context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(containerTokenIdentifier)) {
    // Is the container being relaunched? Or RPC layer let startCall with
    // tokens generated off old-secret through?
    unauthorized = true;
    messageBuilder.append("\n Attempt to relaunch the same ")
      .append("container with id ").append(containerIDStr).append(".");
  } else if (containerTokenIdentifier.getExpiryTimeStamp() < System
    .currentTimeMillis()) {
    // Ensure the token is not expired.
    unauthorized = true;
    messageBuilder.append("\nThis token is expired. current time is ")
      .append(System.currentTimeMillis()).append(" found ")
      .append(containerTokenIdentifier.getExpiryTimeStamp());
    messageBuilder.append("\nNote: System times on machines may be out of sync.")
      .append(" Check system time and time zones.");
  }
  if (unauthorized) {
    String msg = messageBuilder.toString();
    LOG.error(msg);
    throw RPCUtil.getRemoteException(msg);
  }
}
Example 12
Source File: ContainerManagerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * @param containerTokenIdentifier
 *          of the container to be started
 * @throws YarnException
 */
@Private
@VisibleForTesting
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
    ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {

  ContainerId containerId = containerTokenIdentifier.getContainerID();
  String containerIDStr = containerId.toString();
  boolean unauthorized = false;
  StringBuilder messageBuilder =
      new StringBuilder("Unauthorized request to start container. ");
  if (!nmTokenIdentifier.getApplicationAttemptId().getApplicationId().
      equals(containerId.getApplicationAttemptId().getApplicationId())) {
    unauthorized = true;
    messageBuilder.append("\nNMToken for application attempt : ")
      .append(nmTokenIdentifier.getApplicationAttemptId())
      .append(" was used for starting container with container token")
      .append(" issued for application attempt : ")
      .append(containerId.getApplicationAttemptId());
  } else if (!this.context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(containerTokenIdentifier)) {
    // Is the container being relaunched? Or RPC layer let startCall with
    // tokens generated off old-secret through?
    unauthorized = true;
    messageBuilder.append("\n Attempt to relaunch the same ")
      .append("container with id ").append(containerIDStr).append(".");
  } else if (containerTokenIdentifier.getExpiryTimeStamp() < System
    .currentTimeMillis()) {
    // Ensure the token is not expired.
    unauthorized = true;
    messageBuilder.append("\nThis token is expired. current time is ")
      .append(System.currentTimeMillis()).append(" found ")
      .append(containerTokenIdentifier.getExpiryTimeStamp());
    messageBuilder.append("\nNote: System times on machines may be out of sync.")
      .append(" Check system time and time zones.");
  }
  if (unauthorized) {
    String msg = messageBuilder.toString();
    LOG.error(msg);
    throw RPCUtil.getRemoteException(msg);
  }
}