org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerManagerRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #2
Source File: TestRPC.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #3
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #4
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void storeContainer(ContainerId containerId,
    StartContainerRequest startRequest) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("storeContainer: containerId= " + containerId
        + ", startRequest= " + startRequest);
  }

  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_REQUEST_KEY_SUFFIX;
  try {
    db.put(bytes(key),
      ((StartContainerRequestPBImpl) startRequest).getProto().toByteArray());
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 
Example #5
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #6
Source File: TestRPC.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #7
Source File: TestContainerManagerRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #8
Source File: AMLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #9
Source File: StartContainersRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void initLocalRequests() {
  StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<StartContainerRequestProto> requestList =
      p.getStartContainerRequestList();
  this.requests = new ArrayList<StartContainerRequest>();
  for (StartContainerRequestProto r : requestList) {
    this.requests.add(convertFromProtoFormat(r));
  }
}
 
Example #10
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #11
Source File: TestContainerLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #12
Source File: StartContainersRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void addLocalRequestsToProto() {
  maybeInitBuilder();
  builder.clearStartContainerRequest();
  List<StartContainerRequestProto> protoList =
      new ArrayList<StartContainerRequestProto>();
  for (StartContainerRequest r : this.requests) {
    protoList.add(convertToProtoFormat(r));
  }
  builder.addAllStartContainerRequest(protoList);
}
 
Example #13
Source File: StartContainersRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void initLocalRequests() {
  StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<StartContainerRequestProto> requestList =
      p.getStartContainerRequestList();
  this.requests = new ArrayList<StartContainerRequest>();
  for (StartContainerRequestProto r : requestList) {
    this.requests.add(convertFromProtoFormat(r));
  }
}
 
Example #14
Source File: StartContainersRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setStartContainerRequests(List<StartContainerRequest> requests) {
  maybeInitBuilder();
  if (requests == null) {
    builder.clearStartContainerRequest();
  }
  this.requests = requests;
}
 
Example #15
Source File: StartContainersRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public List<StartContainerRequest> getStartContainerRequests() {
  if (this.requests != null) {
    return this.requests;
  }
  initLocalRequests();
  return this.requests;
}
 
Example #16
Source File: ContainerManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());
  
  Set<Integer> cores= this.context.getCoresManager().allocateCores(containerId, 
  		                          token.getResource().getVirtualCores());
 
  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(this.context,getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled(),cores);
    
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
 
Example #17
Source File: NMLeveldbStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void storeContainer(ContainerId containerId,
    StartContainerRequest startRequest) throws IOException {
  String key = CONTAINERS_KEY_PREFIX + containerId.toString()
      + CONTAINER_REQUEST_KEY_SUFFIX;
  try {
    db.put(bytes(key),
      ((StartContainerRequestPBImpl) startRequest).getProto().toByteArray());
  } catch (DBException e) {
    throw new IOException(e);
  }
}
 
Example #18
Source File: NMMemoryStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void storeContainer(ContainerId containerId,
    StartContainerRequest startRequest) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.startRequest = startRequest;
  containerStates.put(containerId, rcs);
}
 
Example #19
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #20
Source File: TestApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #21
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #22
Source File: NMMemoryStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void storeContainer(ContainerId containerId,
    StartContainerRequest startRequest) throws IOException {
  RecoveredContainerState rcs = new RecoveredContainerState();
  rcs.startRequest = startRequest;
  containerStates.put(containerId, rcs);
}
 
Example #23
Source File: StartContainersRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void addLocalRequestsToProto() {
  maybeInitBuilder();
  builder.clearStartContainerRequest();
  List<StartContainerRequestProto> protoList =
      new ArrayList<StartContainerRequestProto>();
  for (StartContainerRequest r : this.requests) {
    protoList.add(convertToProtoFormat(r));
  }
  builder.addAllStartContainerRequest(protoList);
}
 
Example #24
Source File: ContainerManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());

  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled());
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
 
Example #25
Source File: StartContainersRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void setStartContainerRequests(List<StartContainerRequest> requests) {
  maybeInitBuilder();
  if (requests == null) {
    builder.clearStartContainerRequest();
  }
  this.requests = requests;
}
 
Example #26
Source File: StartContainersRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public List<StartContainerRequest> getStartContainerRequests() {
  if (this.requests != null) {
    return this.requests;
  }
  initLocalRequests();
  return this.requests;
}
 
Example #27
Source File: TestRPC.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 
Example #28
Source File: NMStateStoreService.java    From big-c with Apache License 2.0 4 votes vote down vote up
public StartContainerRequest getStartRequest() {
  return startRequest;
}
 
Example #29
Source File: TestLogAggregationService.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(localLogDir, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(localLogDir, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(localLogDir, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 
Example #30
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}