org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRPC.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #2
Source File: TestContainerManagerRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #3
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #4
Source File: TestContainerManagerRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #5
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #6
Source File: TestRPC.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #7
Source File: TestContainerLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #8
Source File: TestAMAuthorization.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}
 
Example #9
Source File: ContainerManagementProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #10
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #11
Source File: TestApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #12
Source File: TestAMAuthorization.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}
 
Example #13
Source File: AMLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #14
Source File: ContainerManagementProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #15
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #16
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #17
Source File: TestContainerLaunchRPC.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}
 
Example #18
Source File: TestContainerLaunchRPC.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}
 
Example #19
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 
Example #20
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleContainersLaunch() throws Exception {
  containerManager.start();

  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  for (int i = 0; i < 10; i++) {
    ContainerId cId = createContainerId(i);
    long identifier = 0;
    if ((i & 1) == 0)
      // container with even id fail
      identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
    else
      identifier = DUMMY_RM_IDENTIFIER;
    Token containerToken =
        createContainerToken(cId, identifier, context.getNodeId(), user,
          context.getContainerTokenSecretManager());
    StartContainerRequest request =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    list.add(request);
  }
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerManager.startContainers(requestList);

  Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
  for (ContainerId id : response.getSuccessfullyStartedContainers()) {
    // Containers with odd id should succeed.
    Assert.assertEquals(1, id.getContainerId() & 1);
  }
  Assert.assertEquals(5, response.getFailedRequests().size());
  for (Map.Entry<ContainerId, SerializedException> entry : response
    .getFailedRequests().entrySet()) {
    // Containers with even id should fail.
    Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
    Assert.assertTrue(entry.getValue().getMessage()
      .contains(
        "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
  }
}
 
Example #21
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 
Example #22
Source File: TestRPC.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 
Example #23
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testStartContainerFailureWithUnknownAuxService() throws Exception {
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "existService" });
  conf.setClass(
      String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "existService"),
      ServiceA.class, Service.class);
  containerManager.start();

  List<StartContainerRequest> startRequest =
      new ArrayList<StartContainerRequest>();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
  String serviceName = "non_exist_auxService";
  serviceData.put(serviceName, ByteBuffer.wrap(serviceName.getBytes()));
  containerLaunchContext.setServiceData(serviceData);

  ContainerId cId = createContainerId(0);
  String user = "start_container_fail";
  Token containerToken =
      createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager());
  StartContainerRequest request =
      StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);

  // start containers
  startRequest.add(request);
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(startRequest);

  StartContainersResponse response =
      containerManager.startContainers(requestList);
  Assert.assertTrue(response.getFailedRequests().size() == 1);
  Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
  Assert.assertTrue(response.getFailedRequests().containsKey(cId));
  Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
      .contains("The auxService:" + serviceName + " does not exist"));
}
 
Example #24
Source File: Application.java    From big-c with Apache License 2.0 4 votes vote down vote up
private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}
 
Example #25
Source File: ContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerRemoteLaunchEvent event) {
  LOG.info("Launching " + taskAttemptID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(taskAttemptID, 
        "Container was killed before it was launched");
    return;
  }
  
  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerMgrAddress, containerID);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          event.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startRequest);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(requestList);
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerID)) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }
    ByteBuffer portInfo =
        response.getAllServicesMetaData().get(
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
    int port = -1;
    if(portInfo != null) {
      port = ShuffleHandler.deserializeMetaData(portInfo);
    }
    LOG.info("Shuffle port returned by ContainerManager for "
        + taskAttemptID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + taskAttemptID);
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(taskAttemptID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 
Example #26
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testHandle() throws Exception {
  LOG.info("STARTING testHandle");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);
  String cmAddress = "127.0.0.1:8000";
  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).startContainers(any(StartContainersRequest.class));
    
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  } finally {
    ut.stop();
  }
}
 
Example #27
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testOutOfOrder() throws Exception {
  LOG.info("STARTING testOutOfOrder");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).stopContainers(any(StopContainersRequest.class));

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).startContainers(any(StartContainersRequest.class));
  } finally {
    ut.stop();
  }
}
 
Example #28
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testMyShutdown() throws Exception {
  LOG.info("in test Shutdown");

  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);

  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent =
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);

    ut.waitForPoolToIdle();

    verify(mockCM).startContainers(any(StartContainersRequest.class));

    // skip cleanup and make sure stop kills the container

  } finally {
    ut.stop();
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  }
}
 
Example #29
Source File: YarnContainerProxy.java    From incubator-tajo with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) {
  LOG.info("Launching Container with Id: " + containerID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    LOG.error("Container (" + containerID + " was killed before it was launched");
    return;
  }

  ContainerManagementProtocol proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext = createContainerLaunchContext(commonContainerLaunchContext);

    // Now launch the actual container
    List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>();
    StartContainerRequest startRequest = Records
        .newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(containerLaunchContext);
    startRequestList.add(startRequest);
    StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class);
    startRequests.setStartContainerRequests(startRequestList);
    StartContainersResponse response = proxy.startContainers(startRequests);

    ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID);

    if(portInfo != null) {
      port = PullServerAuxService.deserializeMetaData(portInfo);
    }

    LOG.info("PullServer port returned by ContainerManager for "
        + containerID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + containerID);
    }

    this.state = ContainerState.RUNNING;
    this.hostName = containerMgrAddress.split(":")[0];
    context.getResourceAllocator().addContainer(containerID, this);
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    LOG.error(message);
  } finally {
    if (proxy != null) {
      yarnRPC.stopProxy(proxy, conf);
    }
  }
}
 
Example #30
Source File: ContainerLauncherImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(NMCommunicatorLaunchRequestEvent event) {
  LOG.info("Launching Container with Id: " + event.getContainerId());
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(event.getContainerId(),
        "Container was killed before it was launched");
    return;
  }

  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest = Records
      .newRecord(StartContainerRequest.class);
    startRequest.setContainerToken(event.getContainerToken());
    startRequest.setContainerLaunchContext(containerLaunchContext);

    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(
            StartContainersRequest.newInstance(
                Collections.singletonList(startRequest)));
    if (response.getFailedRequests() != null
        && !response.getFailedRequests().isEmpty()) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new AMContainerEventLaunched(containerID));
    ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
        containerID, clock.getTime(), context.getApplicationAttemptId());
    context.getHistoryHandler().handle(new DAGHistoryEvent(
        null, lEvt));

    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(containerID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}