Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example 2
Source File: TestContainerManagerRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example 3
Source File: TestContainerManagerRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example 4
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example 5
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example 6
Source File: Application.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}
 
Example 7
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleContainersLaunch() throws Exception {
  containerManager.start();

  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  for (int i = 0; i < 10; i++) {
    ContainerId cId = createContainerId(i);
    long identifier = 0;
    if ((i & 1) == 0)
      // container with even id fail
      identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
    else
      identifier = DUMMY_RM_IDENTIFIER;
    Token containerToken =
        createContainerToken(cId, identifier, context.getNodeId(), user,
          context.getContainerTokenSecretManager());
    StartContainerRequest request =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    list.add(request);
  }
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerManager.startContainers(requestList);

  Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
  for (ContainerId id : response.getSuccessfullyStartedContainers()) {
    // Containers with odd id should succeed.
    Assert.assertEquals(1, id.getContainerId() & 1);
  }
  Assert.assertEquals(5, response.getFailedRequests().size());
  for (Map.Entry<ContainerId, SerializedException> entry : response
    .getFailedRequests().entrySet()) {
    // Containers with even id should fail.
    Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
    Assert.assertTrue(entry.getValue().getMessage()
      .contains(
        "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
  }
}
 
Example 8
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 
Example 9
Source File: TestLogAggregationService.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(localLogDir, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(localLogDir, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(localLogDir, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 
Example 10
Source File: TestContainerLaunchRPC.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testRPCTimeout(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);

  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class,
      new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  try {

    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(
        ContainerManagementProtocol.class,
        server.getListenerAddress(), conf);
    ContainerLaunchContext containerLaunchContext = recordFactory
        .newRecordInstance(ContainerLaunchContext.class);

    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId =
        ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId =
        ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier =
        new ContainerTokenIdentifier(containerId, "localhost", "user",
          resource, System.currentTimeMillis() + 10000, 42, 42,
          Priority.newInstance(0), 0);
    Token containerToken =
        TestRPC.newContainerToken(nodeId, "password".getBytes(),
          containerTokenIdentifier);

    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests =
        StartContainersRequest.newInstance(list);
    try {
      proxy.startContainers(allRequests);
    } catch (Exception e) {
      LOG.info(StringUtils.stringifyException(e));
      Assert.assertEquals("Error, exception is not: "
          + SocketTimeoutException.class.getName(),
          SocketTimeoutException.class.getName(), e.getClass().getName());
      return;
    }
  } finally {
    server.stop();
  }

  Assert.fail("timeout exception should have occurred!");
}
 
Example 11
Source File: TestRPC.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 
Example 12
Source File: ContainerLauncherImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerRemoteLaunchEvent event) {
  LOG.info("Launching " + taskAttemptID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(taskAttemptID, 
        "Container was killed before it was launched");
    return;
  }
  
  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerMgrAddress, containerID);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          event.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startRequest);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(requestList);
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerID)) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }
    ByteBuffer portInfo =
        response.getAllServicesMetaData().get(
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
    int port = -1;
    if(portInfo != null) {
      port = ShuffleHandler.deserializeMetaData(portInfo);
    }
    LOG.info("Shuffle port returned by ContainerManager for "
        + taskAttemptID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + taskAttemptID);
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(taskAttemptID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 
Example 13
Source File: TestRPC.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 
Example 14
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testStartContainerFailureWithUnknownAuxService() throws Exception {
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "existService" });
  conf.setClass(
      String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "existService"),
      ServiceA.class, Service.class);
  containerManager.start();

  List<StartContainerRequest> startRequest =
      new ArrayList<StartContainerRequest>();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
  String serviceName = "non_exist_auxService";
  serviceData.put(serviceName, ByteBuffer.wrap(serviceName.getBytes()));
  containerLaunchContext.setServiceData(serviceData);

  ContainerId cId = createContainerId(0);
  String user = "start_container_fail";
  Token containerToken =
      createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager());
  StartContainerRequest request =
      StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);

  // start containers
  startRequest.add(request);
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(startRequest);

  StartContainersResponse response =
      containerManager.startContainers(requestList);
  Assert.assertTrue(response.getFailedRequests().size() == 1);
  Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
  Assert.assertTrue(response.getFailedRequests().containsKey(cId));
  Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
      .contains("The auxService:" + serviceName + " does not exist"));
}
 
Example 15
Source File: Application.java    From big-c with Apache License 2.0 4 votes vote down vote up
private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}
 
Example 16
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleContainersLaunch() throws Exception {
  containerManager.start();

  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  for (int i = 0; i < 10; i++) {
    ContainerId cId = createContainerId(i);
    long identifier = 0;
    if ((i & 1) == 0)
      // container with even id fail
      identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
    else
      identifier = DUMMY_RM_IDENTIFIER;
    Token containerToken =
        createContainerToken(cId, identifier, context.getNodeId(), user,
          context.getContainerTokenSecretManager());
    StartContainerRequest request =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    list.add(request);
  }
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerManager.startContainers(requestList);

  Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
  for (ContainerId id : response.getSuccessfullyStartedContainers()) {
    // Containers with odd id should succeed.
    Assert.assertEquals(1, id.getContainerId() & 1);
  }
  Assert.assertEquals(5, response.getFailedRequests().size());
  for (Map.Entry<ContainerId, SerializedException> entry : response
    .getFailedRequests().entrySet()) {
    // Containers with even id should fail.
    Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
    Assert.assertTrue(entry.getValue().getMessage()
      .contains(
        "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
  }
}
 
Example 17
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 
Example 18
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 
Example 19
Source File: TestLogAggregationService.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(localLogDir, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(localLogDir, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(localLogDir, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 
Example 20
Source File: ContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerRemoteLaunchEvent event) {
  LOG.info("Launching " + taskAttemptID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(taskAttemptID, 
        "Container was killed before it was launched");
    return;
  }
  
  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerMgrAddress, containerID);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          event.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startRequest);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(requestList);
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerID)) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }
    ByteBuffer portInfo =
        response.getAllServicesMetaData().get(
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
    int port = -1;
    if(portInfo != null) {
      port = ShuffleHandler.deserializeMetaData(portInfo);
    }
    LOG.info("Shuffle port returned by ContainerManager for "
        + taskAttemptID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + taskAttemptID);
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(taskAttemptID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}