org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRPC.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #2
Source File: TestContainerManagerRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #3
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #4
Source File: TestRPC.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}
 
Example #5
Source File: TestContainerManagerRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #6
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #7
Source File: TestAMAuthorization.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}
 
Example #8
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #9
Source File: TestApplicationMasterLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #10
Source File: ContainerManagementProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #11
Source File: TestContainerLaunchRPC.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}
 
Example #12
Source File: TestContainerLaunchRPC.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}
 
Example #13
Source File: ContainerManagementProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #14
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #15
Source File: TestContainerLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}
 
Example #16
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #17
Source File: TestAMAuthorization.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}
 
Example #18
Source File: AMLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #19
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testOutOfOrder() throws Exception {
  LOG.info("STARTING testOutOfOrder");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).stopContainers(any(StopContainersRequest.class));

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).startContainers(any(StartContainersRequest.class));
  } finally {
    ut.stop();
  }
}
 
Example #20
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testHandle() throws Exception {
  LOG.info("STARTING testHandle");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);
  String cmAddress = "127.0.0.1:8000";
  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).startContainers(any(StartContainersRequest.class));
    
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  } finally {
    ut.stop();
  }
}
 
Example #21
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testMyShutdown() throws Exception {
  LOG.info("in test Shutdown");

  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);

  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent =
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);

    ut.waitForPoolToIdle();

    verify(mockCM).startContainers(any(StartContainersRequest.class));

    // skip cleanup and make sure stop kills the container

  } finally {
    ut.stop();
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  }
}
 
Example #22
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test(timeout = 5000)
public void testContainerCleaned() throws Exception {
  LOG.info("STARTING testContainerCleaned");
  
  CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
  CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);

  AppContext mockContext = mock(AppContext.class);
  
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      new ContainerManagerForTest(startLaunchBarrier, completeLaunchBarrier);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    
   
    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    startLaunchBarrier.await();
    
         
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);

    completeLaunchBarrier.await();
   
    ut.waitForPoolToIdle();
    
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler, atLeast(2)).handle(arg.capture());
    boolean containerCleaned = false;
    
    for (int i =0; i < arg.getAllValues().size(); i++) {
      LOG.info(arg.getAllValues().get(i).toString());
      Event currentEvent = arg.getAllValues().get(i);
      if (currentEvent.getType() == TaskAttemptEventType.TA_CONTAINER_CLEANED) {
        containerCleaned = true;
      }
    }
    assert(containerCleaned);
    
  } finally {
    ut.stop();
  }
}
 
Example #23
Source File: YarnContainerProxy.java    From incubator-tajo with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) {
  LOG.info("Launching Container with Id: " + containerID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    LOG.error("Container (" + containerID + " was killed before it was launched");
    return;
  }

  ContainerManagementProtocol proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext = createContainerLaunchContext(commonContainerLaunchContext);

    // Now launch the actual container
    List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>();
    StartContainerRequest startRequest = Records
        .newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(containerLaunchContext);
    startRequestList.add(startRequest);
    StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class);
    startRequests.setStartContainerRequests(startRequestList);
    StartContainersResponse response = proxy.startContainers(startRequests);

    ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID);

    if(portInfo != null) {
      port = PullServerAuxService.deserializeMetaData(portInfo);
    }

    LOG.info("PullServer port returned by ContainerManager for "
        + containerID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + containerID);
    }

    this.state = ContainerState.RUNNING;
    this.hostName = containerMgrAddress.split(":")[0];
    context.getResourceAllocator().addContainer(containerID, this);
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    LOG.error(message);
  } finally {
    if (proxy != null) {
      yarnRPC.stopProxy(proxy, conf);
    }
  }
}
 
Example #24
Source File: ContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerRemoteLaunchEvent event) {
  LOG.info("Launching " + taskAttemptID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(taskAttemptID, 
        "Container was killed before it was launched");
    return;
  }
  
  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerMgrAddress, containerID);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          event.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startRequest);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(requestList);
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerID)) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }
    ByteBuffer portInfo =
        response.getAllServicesMetaData().get(
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
    int port = -1;
    if(portInfo != null) {
      port = ShuffleHandler.deserializeMetaData(portInfo);
    }
    LOG.info("Shuffle port returned by ContainerManager for "
        + taskAttemptID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + taskAttemptID);
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(taskAttemptID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 
Example #25
Source File: ContainerLauncherImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(NMCommunicatorLaunchRequestEvent event) {
  LOG.info("Launching Container with Id: " + event.getContainerId());
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(event.getContainerId(),
        "Container was killed before it was launched");
    return;
  }

  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest = Records
      .newRecord(StartContainerRequest.class);
    startRequest.setContainerToken(event.getContainerToken());
    startRequest.setContainerLaunchContext(containerLaunchContext);

    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(
            StartContainersRequest.newInstance(
                Collections.singletonList(startRequest)));
    if (response.getFailedRequests() != null
        && !response.getFailedRequests().isEmpty()) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new AMContainerEventLaunched(containerID));
    ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
        containerID, clock.getTime(), context.getApplicationAttemptId());
    context.getHistoryHandler().handle(new DAGHistoryEvent(
        null, lEvt));

    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(containerID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 
Example #26
Source File: TezContainerLauncherImpl.java    From tez with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchRequest event) {
  LOG.info("Launching " + event.getContainerId());
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(event.getContainerId(),
        "Container was killed before it was launched");
    return;
  }

  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest = Records
      .newRecord(StartContainerRequest.class);
    startRequest.setContainerToken(event.getContainerToken());
    startRequest.setContainerLaunchContext(containerLaunchContext);

    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(
            StartContainersRequest.newInstance(
                Collections.singletonList(startRequest)));
    if (response.getFailedRequests() != null
        && !response.getFailedRequests().isEmpty()) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    getContext().containerLaunched(containerID);
    this.state = ContainerState.RUNNING;

    int shufflePort = TezRuntimeUtils.INVALID_PORT;
    Map<String, java.nio.ByteBuffer> servicesMetaData = response.getAllServicesMetaData();
    if (servicesMetaData != null) {
      String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
          TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
      ByteBuffer portInfo = servicesMetaData.get(auxiliaryService);
      if (portInfo != null) {
        DataInputByteBuffer in = new DataInputByteBuffer();
        in.reset(portInfo);
        shufflePort = in.readInt();
      } else {
        LOG.warn("Shuffle port for {} is not present is the services metadata response", auxiliaryService);
      }
    } else {
      LOG.warn("Shuffle port cannot be found since services metadata response is missing");
    }
    if (deletionTracker != null) {
      deletionTracker.addNodeShufflePort(event.getNodeId(), shufflePort);
    }
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + ExceptionUtils.getStackTrace(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(containerID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}
 
Example #27
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testStartContainerFailureWithUnknownAuxService() throws Exception {
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "existService" });
  conf.setClass(
      String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "existService"),
      ServiceA.class, Service.class);
  containerManager.start();

  List<StartContainerRequest> startRequest =
      new ArrayList<StartContainerRequest>();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
  String serviceName = "non_exist_auxService";
  serviceData.put(serviceName, ByteBuffer.wrap(serviceName.getBytes()));
  containerLaunchContext.setServiceData(serviceData);

  ContainerId cId = createContainerId(0);
  String user = "start_container_fail";
  Token containerToken =
      createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager());
  StartContainerRequest request =
      StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);

  // start containers
  startRequest.add(request);
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(startRequest);

  StartContainersResponse response =
      containerManager.startContainers(requestList);
  Assert.assertTrue(response.getFailedRequests().size() == 1);
  Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
  Assert.assertTrue(response.getFailedRequests().containsKey(cId));
  Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
      .contains("The auxService:" + serviceName + " does not exist"));
}
 
Example #28
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleContainersLaunch() throws Exception {
  containerManager.start();

  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  for (int i = 0; i < 10; i++) {
    ContainerId cId = createContainerId(i);
    long identifier = 0;
    if ((i & 1) == 0)
      // container with even id fail
      identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
    else
      identifier = DUMMY_RM_IDENTIFIER;
    Token containerToken =
        createContainerToken(cId, identifier, context.getNodeId(), user,
          context.getContainerTokenSecretManager());
    StartContainerRequest request =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    list.add(request);
  }
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerManager.startContainers(requestList);

  Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
  for (ContainerId id : response.getSuccessfullyStartedContainers()) {
    // Containers with odd id should succeed.
    Assert.assertEquals(1, id.getContainerId() & 1);
  }
  Assert.assertEquals(5, response.getFailedRequests().size());
  for (Map.Entry<ContainerId, SerializedException> entry : response
    .getFailedRequests().entrySet()) {
    // Containers with even id should fail.
    Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
    Assert.assertTrue(entry.getValue().getMessage()
      .contains(
        "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
  }
}
 
Example #29
Source File: TestContainerLauncherImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test(timeout = 5000)
public void testContainerCleaned() throws Exception {
  LOG.info("STARTING testContainerCleaned");
  
  CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
  CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);

  AppContext mockContext = mock(AppContext.class);
  
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      new ContainerManagerForTest(startLaunchBarrier, completeLaunchBarrier);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    
   
    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    startLaunchBarrier.await();
    
         
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);

    completeLaunchBarrier.await();
   
    ut.waitForPoolToIdle();
    
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler, atLeast(2)).handle(arg.capture());
    boolean containerCleaned = false;
    
    for (int i =0; i < arg.getAllValues().size(); i++) {
      LOG.info(arg.getAllValues().get(i).toString());
      Event currentEvent = arg.getAllValues().get(i);
      if (currentEvent.getType() == TaskAttemptEventType.TA_CONTAINER_CLEANED) {
        containerCleaned = true;
      }
    }
    assert(containerCleaned);
    
  } finally {
    ut.stop();
  }
}
 
Example #30
Source File: TestContainerLauncherImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testMyShutdown() throws Exception {
  LOG.info("in test Shutdown");

  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);

  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent =
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);

    ut.waitForPoolToIdle();

    verify(mockCM).startContainers(any(StartContainersRequest.class));

    // skip cleanup and make sure stop kills the container

  } finally {
    ut.stop();
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  }
}