Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newContainerLaunchContext()

private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
 

private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
 

protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
    throws IOException {
  try {
    Thread.sleep(1000);
  } catch (InterruptedException e) {
    Thread.currentThread().interrupt();
  }
  return BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
      Arrays.asList("sleep", "60000"), Collections.emptyMap(), null, Collections.emptyMap());
}
 

@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);
  InlineDispatcher rmDispatcher = new InlineDispatcher();

  ContainerAllocationExpirer containerAllocationExpirer =
      mock(ContainerAllocationExpirer.class);
  amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  amFinishingMonitor = mock(AMLivelinessMonitor.class);
  writer = mock(RMApplicationHistoryWriter.class);
  MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey();
  when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData);
  rmContext =
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        null, amRMTokenManager,
        new RMContainerTokenSecretManager(conf),
        nmTokenManager,
        clientToAMTokenManager,
        writer);
  
  store = mock(RMStateStore.class);
  ((RMContextImpl) rmContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher);
  
  scheduler = mock(YarnScheduler.class);
  masterService = mock(ApplicationMasterService.class);
  applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
  
  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher());

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher());
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(AMLauncherEventType.class, 
      new TestAMLauncherEventDispatcher());

  rmnodeEventHandler = mock(RMNodeImpl.class);
  rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);

  rmDispatcher.init(conf);
  rmDispatcher.start();
  

  ApplicationId applicationId = MockApps.newAppID(appId++);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);

  resourceScheduler = mock(ResourceScheduler.class);

  ApplicationResourceUsageReport appResUsgRpt =
      mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
  when(appResUsgRpt.getGcoreSeconds()).thenReturn(0L);
  when(resourceScheduler
      .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
   .thenReturn(appResUsgRpt);
  spyRMContext = spy(rmContext);
  Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();


  final String user = MockApps.newUserName();
  final String queue = MockApps.newQueue();
  submissionContext = mock(ApplicationSubmissionContext.class);
  when(submissionContext.getQueue()).thenReturn(queue);
  Resource resource = BuilderUtils.newResource(1536, 1, 1);
  ContainerLaunchContext amContainerSpec =
      BuilderUtils.newContainerLaunchContext(null, null,
          null, null, null, null);
  when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
  when(submissionContext.getResource()).thenReturn(resource);

  unmanagedAM = false;
  
  application = mock(RMAppImpl.class);
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
          masterService, submissionContext, new Configuration(), false,
          BuilderUtils.newResourceRequest(
              RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
              submissionContext.getResource(), 1));

  when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
  when(application.getApplicationId()).thenReturn(applicationId);
  spyRMContext.getRMApps().put(application.getApplicationId(), application);

  testAppAttemptNewState();
}
 

@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);
  InlineDispatcher rmDispatcher = new InlineDispatcher();

  ContainerAllocationExpirer containerAllocationExpirer =
      mock(ContainerAllocationExpirer.class);
  amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  amFinishingMonitor = mock(AMLivelinessMonitor.class);
  writer = mock(RMApplicationHistoryWriter.class);
  MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey();
  when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData);
  rmContext =
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        null, amRMTokenManager,
        new RMContainerTokenSecretManager(conf),
        nmTokenManager,
        clientToAMTokenManager,
        writer);
  
  store = mock(RMStateStore.class);
  ((RMContextImpl) rmContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher);
  
  scheduler = mock(YarnScheduler.class);
  masterService = mock(ApplicationMasterService.class);
  applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
  
  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher());

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher());
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(AMLauncherEventType.class, 
      new TestAMLauncherEventDispatcher());

  rmnodeEventHandler = mock(RMNodeImpl.class);
  rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);

  rmDispatcher.init(conf);
  rmDispatcher.start();
  

  ApplicationId applicationId = MockApps.newAppID(appId++);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);

  resourceScheduler = mock(ResourceScheduler.class);

  ApplicationResourceUsageReport appResUsgRpt =
      mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
  when(resourceScheduler
      .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
   .thenReturn(appResUsgRpt);
  spyRMContext = spy(rmContext);
  Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();


  final String user = MockApps.newUserName();
  final String queue = MockApps.newQueue();
  submissionContext = mock(ApplicationSubmissionContext.class);
  when(submissionContext.getQueue()).thenReturn(queue);
  Resource resource = BuilderUtils.newResource(1536, 1);
  ContainerLaunchContext amContainerSpec =
      BuilderUtils.newContainerLaunchContext(null, null,
          null, null, null, null);
  when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
  when(submissionContext.getResource()).thenReturn(resource);

  unmanagedAM = false;
  
  application = mock(RMAppImpl.class);
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
          masterService, submissionContext, new Configuration(), false,
          BuilderUtils.newResourceRequest(
              RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
              submissionContext.getResource(), 1));

  when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
  when(application.getApplicationId()).thenReturn(applicationId);
  spyRMContext.getRMApps().put(application.getApplicationId(), application);

  testAppAttemptNewState();
}
 

@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

public ContainerLaunchContext createContainerLaunchContext(ContainerLaunchContext commonContainerLaunchContext) {
  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerLaunchContext.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Map.Entry<String, ByteBuffer> entry : commonContainerLaunchContext.getServiceData().entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  ////////////////////////////////////////////////////////////////////////////
  // Set the local resources
  ////////////////////////////////////////////////////////////////////////////
  // Set the necessary command to execute the application master
  Vector<CharSequence> vargs = new Vector<CharSequence>(30);

  // Set java executable command
  //LOG.info("Setting up app master command");
  vargs.add("${JAVA_HOME}" + "/bin/java");
  // Set Xmx based on am memory size
  vargs.add("-Xmx2000m");
  // Set Remote Debugging
  //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) {
  //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005");
  //}
  // Set class name
  //vargs.add(getRunnerClass());
  vargs.add(TajoWorker.class.getCanonicalName());
  vargs.add("tr");     //workerMode
  vargs.add(getId()); // subqueryId
  vargs.add(containerMgrAddress); // nodeId
  vargs.add(containerID.toString()); // containerId
  Vector<CharSequence> taskParams = getTaskParams();
  if(taskParams != null) {
    vargs.addAll(taskParams);
  }

  vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
  vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

  // Get final commmand
  StringBuilder command = new StringBuilder();
  for (CharSequence str : vargs) {
    command.append(str).append(" ");
  }

  LOG.info("Completed setting up TaskRunner command " + command.toString());
  List<String> commands = new ArrayList<String>();
  commands.add(command.toString());

  return BuilderUtils.newContainerLaunchContext(commonContainerLaunchContext.getLocalResources(),
      myEnv,
      commands,
      myServiceData,
      null,
      new HashMap<ApplicationAccessType, String>());
}
 

private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException {
  QueryId queryId = queryInProgress.getQueryId();
  ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId);

  LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" +
      queryId + ", appId=" + appId);

  ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);
  // set the application name
  appContext.setApplicationName("Tajo");

  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(5);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  ContainerLaunchContext commonContainerLaunchContext =
      YarnContainerProxy.createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerLaunchContext.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);

  ////////////////////////////////////////////////////////////////////////////
  // Set the local resources
  ////////////////////////////////////////////////////////////////////////////
  // Set the necessary command to execute the application master
  Vector<CharSequence> vargs = new Vector<CharSequence>(30);

  // Set java executable command
  //LOG.info("Setting up app master command");
  vargs.add("${JAVA_HOME}" + "/bin/java");
  // Set Xmx based on am memory size
  String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m");

  for(String eachToken: jvmOptions.split((" "))) {
    vargs.add(eachToken);
  }
  // Set Remote Debugging
  //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) {
  //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005");
  //}
  // Set class name
  vargs.add(TajoWorker.class.getCanonicalName());
  vargs.add("qm");
  vargs.add(queryId.toString()); // queryId
  vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" +
      masterContext.getTajoMasterService().getBindAddress().getPort());

  vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
  vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

  // Get final commmand
  StringBuilder command = new StringBuilder();
  for (CharSequence str : vargs) {
    command.append(str).append(" ");
  }

  LOG.info("Completed setting up QueryMasterRunner command " + command.toString());
  List<String> commands = new ArrayList<String>();
  commands.add(command.toString());

  final Resource resource = Records.newRecord(Resource.class);
  // TODO - get default value from conf
  resource.setMemory(2000);
  resource.setVirtualCores(1);

  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();

  ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext(
      commonContainerLaunchContext.getLocalResources(),
      myEnv,
      commands,
      myServiceData,
      null,
      new HashMap<ApplicationAccessType, String>(2)
  );

  appContext.setAMContainerSpec(masterContainerContext);

  LOG.info("Submitting QueryMaster to ResourceManager");
  yarnClient.submitApplication(appContext);

  ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED));
  ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();

  LOG.info("Launching QueryMaster with appAttemptId: " + attemptId);

  return attemptId;
}
 

protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
    throws IOException {
  return BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "60000"), Collections.emptyMap(), null, Collections.emptyMap());
}