Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerId#getApplicationAttemptId()

private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 

private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 

public static void intercept(@Argument(0) String containerID, @Argument(1) long currentMemUsage,
                             @Argument(3) long limit) throws Exception {
    try {

        ContainerId cID = ConverterUtils.toContainerId(containerID);
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
                .withId(applicationId)
                .withApplicationID(applicationId)
                .withAttemptID(attemptId)
                .withContainerID(containerID)
                .build();

        long memUsage = (currentMemUsage > 0) ? currentMemUsage : 0;

        ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                .setType(ContainerType.MEMORY.name())
                .setValue(memUsage)
                .setLimit(limit)
                .build();
        eventHandler.accept(System.currentTimeMillis(), header, event);
    } catch (Throwable ignored) {
    }
}
 

public static void intercept(@This Object containerMetrics,
                             @Argument(1) int milliVcoresUsed) throws Exception {
    try {
        if (getField() != null) {
            ContainerId cID = (ContainerId) getField().get(containerMetrics);
            ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
            String applicationId = applicationAttemptId.getApplicationId().toString();
            String attemptId = applicationAttemptId.toString();

            float cpuVcoreUsed = (milliVcoresUsed > 0) ? (float) milliVcoresUsed / 1000 : 0f;
            int cpuVcoreLimit = ((ContainerMetrics) containerMetrics).cpuVcoreLimit.value();

            Header header = Header.newBuilder()
                    .withId(applicationId)
                    .withApplicationID(applicationId)
                    .withAttemptID(attemptId)
                    .withContainerID(cID.toString())
                    .build();

            ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                    .setType(ContainerType.VCORE.name())
                    .setValue(cpuVcoreUsed)
                    .setLimit(cpuVcoreLimit)
                    .build();
            eventHandler.accept(System.currentTimeMillis(), header, event);
        } else {
            LOGGER.warn("ContainerMetrics class does not have containerId field");
        }
    } catch (Throwable ignored) {
    }
}
 

private Header.SerializedHeader createCachedHeader() {
    String user = System.getenv(ApplicationConstants.Environment.USER.name());
    String containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
    String host = System.getenv(ApplicationConstants.Environment.NM_HOST.name());

    // Get applicationID
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
    ApplicationId appId = appAttemptID.getApplicationId();

    //build the header for the whole application once
    return Header.newBuilder()
            .withId(appId.toString())
            .addTag(Header.Tag.YARN_APPLICATION.name())
            .addTags(System.getProperty("garmadon.tags"))
            .withHostname(host)
            .withApplicationID(appId.toString())
            .withAttemptID(appAttemptID.toString())
            .withUser(user)
            .withContainerID(containerIdString)
            .withPid(HeaderUtils.getPid())
            .withFramework(framework.toString())
            .withComponent(component.name())
            .withExecutorId(executorId)
            .withMainClass(mainClass)
            .buildSerializedHeader();
}
 

/**
 * @param containerIdString the Container ID of the running Container.
 * @return the Application Attempt ID of the YARN application.
 */
public static ApplicationAttemptId getAppAttemptId(final String containerIdString) {
  if (containerIdString == null) {
    return null;
  }

  try {
    final ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    return containerId.getApplicationAttemptId();
  } catch (Exception e) {
    LOG.log(Level.WARNING, "Unable to get the applicationAttempt ID from the environment, exception " +
        e + " was thrown.");
    return null;
  }
}
 

public YarnAppState(int taskId,
                    ContainerId amContainerId,
                    String nodeHost,
                    int nodePort,
                    int nodeHttpPort
                    ) {
  this.taskId = taskId;
  this.amContainerId = amContainerId;
  this.nodeHost = nodeHost;
  this.nodePort = nodePort;
  this.nodeHttpPort = nodeHttpPort;
  this.appAttemptId = amContainerId.getApplicationAttemptId();
}
 

public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);

    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();

    long appSubmitTime = Long.parseLong(appSubmitTimeStr);

    final Configuration conf = new Configuration(new YarnConfiguration());
    TezUtils.addUserSpecifiedTezConfiguration(conf);

    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());

    // Do not automatically close FileSystem objects so that in case of
    // SIGTERM I have a chance to write out the job history. I'll be closing
    // the objects myself.
    conf.setBoolean("fs.automatic.close", false);

    // Command line options
    Options opts = new Options();
    opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION,
        false, "Run Tez Application Master in Session mode");

    CommandLine cliParser = new GnuParser().parse(opts, args);

    DAGAppMaster appMaster =
        new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), appSubmitTime,
            cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION));
    ShutdownHookManager.get().addShutdownHook(
      new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);

    Limits.setConfiguration(conf);
    initAndStartAppMaster(appMaster, conf,
        jobUserName);

  } catch (Throwable t) {
    LOG.fatal("Error starting DAGAppMaster", t);
    System.exit(1);
  }
}
 

/**
 * @param args
 *          Command line args
 * @throws Throwable
 */
public static void main(final String[] args) throws Throwable
{
  LoggerUtil.setupMDC("master");
  StdOutErrLog.tieSystemOutAndErrToLog();
  LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path"));

  LOG.info("version: {}", VersionInfo.APEX_VERSION.getBuildVersion());
  StringWriter sw = new StringWriter();
  for (Map.Entry<String, String> e : System.getenv().entrySet()) {
    sw.append("\n").append(e.getKey()).append("=").append(e.getValue());
  }
  LOG.info("appmaster env:" + sw.toString());

  Options opts = new Options();
  opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");

  opts.addOption("help", false, "Print usage");
  CommandLine cliParser = new GnuParser().parse(opts, args);

  // option "help" overrides and cancels any run
  if (cliParser.hasOption("help")) {
    new HelpFormatter().printHelp("ApplicationMaster", opts);
    return;
  }

  Map<String, String> envs = System.getenv();
  ApplicationAttemptId appAttemptID = Records.newRecord(ApplicationAttemptId.class);
  if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
    if (cliParser.hasOption("app_attempt_id")) {
      String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException("Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  boolean result = false;
  StreamingAppMasterService appMaster = null;
  try {
    appMaster = new StreamingAppMasterService(appAttemptID);
    LOG.info("Initializing Application Master.");

    Configuration conf = new YarnConfiguration();
    appMaster.init(conf);
    appMaster.start();
    result = appMaster.run();
  } catch (Throwable t) {
    LOG.error("Exiting Application Master", t);
    System.exit(1);
  } finally {
    if (appMaster != null) {
      appMaster.stop();
    }
  }

  if (result) {
    LOG.info("Application Master completed.");
    System.exit(0);
  } else {
    LOG.info("Application Master failed.");
    System.exit(2);
  }
}
 

/**
 * Parse command line options
 *
 * @param args Command line args
 * @return Whether init successful and run should be invoked
 * @throws ParseException
 * @throws IOException
 */
public boolean init(String[] args) throws ParseException, IOException {
  CommandLine cliParser = AMOptions.parse(new GnuParser(), args);

  //Check whether customer log4j.properties file exists
  if (fileExist(log4jPath)) {
    try {
      Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class,
              log4jPath);
    } catch (Exception e) {
      LOG.warn("Can not set up custom log4j properties. " + e);
    }
  }

  if (AMOptions.HELP.has(cliParser)) {
    AMOptions.printHelp();
    return false;
  }


  zkQuorum = AMOptions.ZK_QUORUM.get(cliParser);
  zkRoot = AMOptions.ZK_ROOT.get(cliParser);
  appJarPath = new Path(AMOptions.APP_JAR_PATH.get(cliParser));

  Map<String, String> envs = System.getenv();

  if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
    if (AMOptions.APP_ATTEMPT_ID.has(cliParser)) {
      String appIdStr = AMOptions.APP_ATTEMPT_ID.get(cliParser, "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException(
              "Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs
            .get(Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
    throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_HOST.name())) {
    throw new RuntimeException(Environment.NM_HOST.name()
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
    throw new RuntimeException(Environment.NM_HTTP_PORT
            + " not set in the environment");
  }
  if (!envs.containsKey(Environment.NM_PORT.name())) {
    throw new RuntimeException(Environment.NM_PORT.name()
            + " not set in the environment");
  }

  LOG.info("Application master for app" + ", appId="
          + appAttemptID.getApplicationId().getId() + ", clustertimestamp="
          + appAttemptID.getApplicationId().getClusterTimestamp()
          + ", attemptId=" + appAttemptID.getAttemptId());


  if (cliParser.hasOption("shell_env")) {
    String shellEnvs[] = cliParser.getOptionValues("shell_env");
    for (String env : shellEnvs) {
      env = env.trim();
      int index = env.indexOf('=');
      if (index == -1) {
        shellEnv.put(env, "");
        continue;
      }
      String key = env.substring(0, index);
      String val = "";
      if (index < (env.length() - 1)) {
        val = env.substring(index + 1);
      }
      shellEnv.put(key, val);
    }
  }


  if (envs.containsKey(Constants.TIMELINEDOMAIN)) {
    domainId = envs.get(Constants.TIMELINEDOMAIN);
  }
  return true;
}
 

/**
 * @param args
 *          Command line args
 * @throws Throwable
 */
public static void main(final String[] args) throws Throwable
{
  LoggerUtil.setupMDC("master");
  StdOutErrLog.tieSystemOutAndErrToLog();
  LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path"));

  LOG.info("version: {}", VersionInfo.APEX_VERSION.getBuildVersion());
  StringWriter sw = new StringWriter();
  for (Map.Entry<String, String> e : System.getenv().entrySet()) {
    sw.append("\n").append(e.getKey()).append("=").append(e.getValue());
  }
  LOG.info("appmaster env:" + sw.toString());

  Options opts = new Options();
  opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");

  opts.addOption("help", false, "Print usage");
  CommandLine cliParser = new GnuParser().parse(opts, args);

  // option "help" overrides and cancels any run
  if (cliParser.hasOption("help")) {
    new HelpFormatter().printHelp("ApplicationMaster", opts);
    return;
  }

  Map<String, String> envs = System.getenv();
  ApplicationAttemptId appAttemptID = Records.newRecord(ApplicationAttemptId.class);
  if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
    if (cliParser.hasOption("app_attempt_id")) {
      String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException("Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  boolean result = false;
  StreamingAppMasterService appMaster = null;
  try {
    appMaster = new StreamingAppMasterService(appAttemptID);
    LOG.info("Initializing Application Master.");

    Configuration conf = new YarnConfiguration();
    appMaster.init(conf);
    appMaster.start();
    result = appMaster.run();
  } catch (Throwable t) {
    LOG.error("Exiting Application Master", t);
    System.exit(1);
  } finally {
    if (appMaster != null) {
      appMaster.stop();
    }
  }

  if (result) {
    LOG.info("Application Master completed.");
    System.exit(0);
  } else {
    LOG.info("Application Master failed.");
    System.exit(2);
  }
}
 

public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    
    validateInputParam(containerIdStr,
        Environment.CONTAINER_ID.name());
    validateInputParam(nodeHostString, Environment.NM_HOST.name());
    validateInputParam(nodePortString, Environment.NM_PORT.name());
    validateInputParam(nodeHttpPortString,
        Environment.NM_HTTP_PORT.name());
    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();
    long appSubmitTime = Long.parseLong(appSubmitTimeStr);
    
    
    MRAppMaster appMaster =
        new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), appSubmitTime);
    ShutdownHookManager.get().addShutdownHook(
      new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
    JobConf conf = new JobConf(new YarnConfiguration());
    conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
    
    MRWebAppUtil.initialize(conf);
    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());
    conf.set(MRJobConfig.USER_NAME, jobUserName);
    initAndStartAppMaster(appMaster, conf, jobUserName);
  } catch (Throwable t) {
    LOG.fatal("Error starting MRAppMaster", t);
    ExitUtil.terminate(1, t);
  }
}
 

@Override
public GetContainerReportResponse getContainerReport(
    GetContainerReportRequest request) throws YarnException, IOException {
  ContainerId containerId = request.getContainerId();
  ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  ApplicationId appId = appAttemptId.getApplicationId();
  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    throw RPCUtil.getRemoteException(ie);
  }
  RMApp application = this.rmContext.getRMApps().get(appId);
  if (application == null) {
    // If the RM doesn't have the application, throw
    // ApplicationNotFoundException and let client to handle.
    throw new ApplicationNotFoundException("Application with id '" + appId
        + "' doesn't exist in RM.");
  }
  boolean allowAccess = checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.VIEW_APP, application);
  GetContainerReportResponse response = null;
  if (allowAccess) {
    RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
    if (appAttempt == null) {
      throw new ApplicationAttemptNotFoundException(
          "ApplicationAttempt with id '" + appAttemptId +
          "' doesn't exist in RM.");
    }
    RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer(
        containerId);
    if (rmConatiner == null) {
      throw new ContainerNotFoundException("Container with id '" + containerId
          + "' doesn't exist in RM.");
    }
    response = GetContainerReportResponse.newInstance(rmConatiner
        .createContainerReport());
  } else {
    throw new YarnException("User " + callerUGI.getShortUserName()
        + " does not have privilage to see this aplication " + appId);
  }
  return response;
}
 

@Test
public void testContainerLogs() throws IOException {
  WebResource r = resource();
  final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
  final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
      .toString();
  final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  final ApplicationId appId = appAttemptId.getApplicationId();
  final String appIdStr = appId.toString();
  final String filename = "logfile1";
  final String logMessage = "log message\n";
  nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
      appId, null, nmContext));
  
  MockContainer container = new MockContainer(appAttemptId,
      new AsyncDispatcher(), new Configuration(), "user", appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(containerId, container);
  
  // write out log file
  Path path = dirsHandler.getLogPathForWrite(
      ContainerLaunch.getRelativeContainerLogDir(
          appIdStr, containerIdStr) + "/" + filename, false);
  
  File logFile = new File(path.toUri().getPath());
  logFile.deleteOnExit();
  assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
  PrintWriter pw = new PrintWriter(logFile);
  pw.print(logMessage);
  pw.close();

  // ask for it
  ClientResponse response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path(filename)
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  String responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
  
  // ask for file that doesn't exist
  response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path("uhhh")
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
  responseText = response.getEntity(String.class);
  assertTrue(responseText.contains("Cannot find this log on the local disk."));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(containerId);
  Assert.assertNull(nmContext.getContainers().get(containerId));
  response =
      r.path("ws").path("v1").path("node").path("containerlogs")
          .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
          .get(ClientResponse.class);
  responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
}
 

public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    
    validateInputParam(containerIdStr,
        Environment.CONTAINER_ID.name());
    validateInputParam(nodeHostString, Environment.NM_HOST.name());
    validateInputParam(nodePortString, Environment.NM_PORT.name());
    validateInputParam(nodeHttpPortString,
        Environment.NM_HTTP_PORT.name());
    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();
    long appSubmitTime = Long.parseLong(appSubmitTimeStr);
    
    
    MRAppMaster appMaster =
        new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), appSubmitTime);
    ShutdownHookManager.get().addShutdownHook(
      new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
    JobConf conf = new JobConf(new YarnConfiguration());
    conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
    
    MRWebAppUtil.initialize(conf);
    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());
    conf.set(MRJobConfig.USER_NAME, jobUserName);
    initAndStartAppMaster(appMaster, conf, jobUserName);
  } catch (Throwable t) {
    LOG.fatal("Error starting MRAppMaster", t);
    ExitUtil.terminate(1, t);
  }
}
 

@Override
public GetContainerReportResponse getContainerReport(
    GetContainerReportRequest request) throws YarnException, IOException {
  ContainerId containerId = request.getContainerId();
  ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  ApplicationId appId = appAttemptId.getApplicationId();
  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    throw RPCUtil.getRemoteException(ie);
  }
  RMApp application = this.rmContext.getRMApps().get(appId);
  if (application == null) {
    // If the RM doesn't have the application, throw
    // ApplicationNotFoundException and let client to handle.
    throw new ApplicationNotFoundException("Application with id '" + appId
        + "' doesn't exist in RM.");
  }
  boolean allowAccess = checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.VIEW_APP, application);
  GetContainerReportResponse response = null;
  if (allowAccess) {
    RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
    if (appAttempt == null) {
      throw new ApplicationAttemptNotFoundException(
          "ApplicationAttempt with id '" + appAttemptId +
          "' doesn't exist in RM.");
    }
    RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer(
        containerId);
    if (rmConatiner == null) {
      throw new ContainerNotFoundException("Container with id '" + containerId
          + "' doesn't exist in RM.");
    }
    response = GetContainerReportResponse.newInstance(rmConatiner
        .createContainerReport());
  } else {
    throw new YarnException("User " + callerUGI.getShortUserName()
        + " does not have privilage to see this aplication " + appId);
  }
  return response;
}
 

@Test
public void testContainerLogs() throws IOException {
  WebResource r = resource();
  final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
  final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
      .toString();
  final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  final ApplicationId appId = appAttemptId.getApplicationId();
  final String appIdStr = appId.toString();
  final String filename = "logfile1";
  final String logMessage = "log message\n";
  nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
      appId, null, nmContext));
  
  MockContainer container = new MockContainer(appAttemptId,
      new AsyncDispatcher(), new Configuration(), "user", appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(containerId, container);
  
  // write out log file
  Path path = dirsHandler.getLogPathForWrite(
      ContainerLaunch.getRelativeContainerLogDir(
          appIdStr, containerIdStr) + "/" + filename, false);
  
  File logFile = new File(path.toUri().getPath());
  logFile.deleteOnExit();
  assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
  PrintWriter pw = new PrintWriter(logFile);
  pw.print(logMessage);
  pw.close();

  // ask for it
  ClientResponse response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path(filename)
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  String responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
  
  // ask for file that doesn't exist
  response = r.path("ws").path("v1").path("node")
      .path("containerlogs").path(containerIdStr).path("uhhh")
      .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
  responseText = response.getEntity(String.class);
  assertTrue(responseText.contains("Cannot find this log on the local disk."));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(containerId);
  Assert.assertNull(nmContext.getContainers().get(containerId));
  response =
      r.path("ws").path("v1").path("node").path("containerlogs")
          .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
          .get(ClientResponse.class);
  responseText = response.getEntity(String.class);
  assertEquals(logMessage, responseText);
}
 

public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    final String pid = System.getenv().get("JVM_PID");
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV);
    if (clientVersion == null) {
      clientVersion = VersionInfo.UNKNOWN;
    }

    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();

    long appSubmitTime = Long.parseLong(appSubmitTimeStr);

    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());

    // Command line options
    Options opts = new Options();
    opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION,
        false, "Run Tez Application Master in Session mode");

    CommandLine cliParser = new GnuParser().parse(opts, args);
    boolean sessionModeCliOption = cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION);

    LOG.info("Creating DAGAppMaster for "
        + "applicationId=" + applicationAttemptId.getApplicationId()
        + ", attemptNum=" + applicationAttemptId.getAttemptId()
        + ", AMContainerId=" + containerId
        + ", jvmPid=" + pid
        + ", userFromEnv=" + jobUserName
        + ", cliSessionOption=" + sessionModeCliOption
        + ", pwd=" + System.getenv(Environment.PWD.name())
        + ", localDirs=" + System.getenv(Environment.LOCAL_DIRS.name())
        + ", logDirs=" + System.getenv(Environment.LOG_DIRS.name()));

    // TODO Does this really need to be a YarnConfiguration ?
    Configuration conf = new Configuration(new YarnConfiguration());

    ConfigurationProto confProto =
        TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
    TezUtilsInternal.addUserSpecifiedTezConfiguration(conf, confProto.getConfKeyValuesList());

    AMPluginDescriptorProto amPluginDescriptorProto = null;
    if (confProto.hasAmPluginDescriptor()) {
      amPluginDescriptorProto = confProto.getAmPluginDescriptor();
    }

    UserGroupInformation.setConfiguration(conf);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);

    DAGAppMaster appMaster =
        new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime,
            sessionModeCliOption,
            System.getenv(Environment.PWD.name()),
            TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())),
            TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())),
            clientVersion, credentials, jobUserName, amPluginDescriptorProto);
    ShutdownHookManager.get().addShutdownHook(
      new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);

    // log the system properties
    if (LOG.isInfoEnabled()) {
      String systemPropsToLog = TezCommonUtils.getSystemPropertiesToLog(conf);
      if (systemPropsToLog != null) {
        LOG.info(systemPropsToLog);
      }
    }

    initAndStartAppMaster(appMaster, conf);

  } catch (Throwable t) {
    LOG.error("Error starting DAGAppMaster", t);
    System.exit(1);
  }
}
 

public static void intercept(@This Object rmContainerImpl,
                             @Argument(0) RMContainerEvent rmContainerEvent) {
    try {
        ContainerId cID = rmContainerEvent.getContainerId();
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
            .withId(applicationId)
            .withApplicationID(applicationId)
            .withAttemptID(attemptId)
            .withContainerID(cID.toString())
            .build();

        RMContainerImpl rmc = (RMContainerImpl) rmContainerImpl;

        ResourceManagerEventProtos.ContainerEvent.Builder eventBuilder = ResourceManagerEventProtos.ContainerEvent.newBuilder()
            .setType(rmContainerEvent.getType().name())
            .setState(rmc.getState().name())
            .setStartTime(rmc.getCreationTime())
            .setLogUrl(rmc.getLogURL());

        if (rmc.getContainer() != null && rmc.getContainer().getResource() != null) {
            eventBuilder.setVcoresReserved(rmc.getContainer().getResource().getVirtualCores());
            eventBuilder.setMemoryReserved(rmc.getContainer().getResource().getMemory());
        }

        if (rmc.getAllocatedNode() != null) {
            eventBuilder.setContainerHostname(rmc.getAllocatedNode().getHost());
        }

        ContainerStatus containerStatus = (ContainerStatus) getField().get(rmContainerImpl);
        if (containerStatus != null) {
            eventBuilder
                .setIsFinished(true)
                .setExitStatus(rmc.getContainerExitStatus())
                .setReason(rmc.getDiagnosticsInfo())
                .setFinishTime(rmc.getFinishTime());
        }

        eventHandler.accept(System.currentTimeMillis(), header, eventBuilder.build());
    } catch (Throwable ignored) {
    }
}
 

/**
 * Returns a new RMContainer corresponding to the RMNode and RMContext. The RMContainer is the 
 * ResourceManager's view of an application container per the Hadoop docs
 * 
 * @param node
 * @param context
 * @param appId
 * @param cores
 * @param memory
 * @return RMContainer
 */
public static RMContainer getRMContainer(RMNode node, RMContext context, int appId, int cores, int memory) {
  ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456789, 1), 1), appId);

  Container container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(),
      Resources.createResource(memory, cores), null, null);
  return new RMContainerImpl(container, containerId.getApplicationAttemptId(), node.getNodeID(), "user1", context);
}