Java Code Examples for org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils#newAMInfo()

The following examples show how to use org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils#newAMInfo() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private static AMInfo createAMInfo(int attempt) {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(100, 1), attempt);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  return MRBuilderUtils.newAMInfo(appAttemptId, System.currentTimeMillis(),
      containerId, NM_HOST, NM_PORT, NM_HTTP_PORT);
}
Example 2
Source File: CompletedJob.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public List<AMInfo> getAMInfos() {
  List<AMInfo> amInfos = new LinkedList<AMInfo>();
  for (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.AMInfo jhAmInfo : jobInfo
      .getAMInfos()) {
    AMInfo amInfo =
        MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
            jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
            jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
            jhAmInfo.getNodeManagerHttpPort());
 
    amInfos.add(amInfo);
  }
  return amInfos;
}
Example 3
Source File: MockJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static AMInfo createAMInfo(int attempt) {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(100, 1), attempt);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  return MRBuilderUtils.newAMInfo(appAttemptId, System.currentTimeMillis(),
      containerId, NM_HOST, NM_PORT, NM_HTTP_PORT);
}
Example 4
Source File: CompletedJob.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public List<AMInfo> getAMInfos() {
  List<AMInfo> amInfos = new LinkedList<AMInfo>();
  for (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.AMInfo jhAmInfo : jobInfo
      .getAMInfos()) {
    AMInfo amInfo =
        MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
            jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
            jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
            jhAmInfo.getNodeManagerHttpPort());
 
    amInfos.add(amInfo);
  }
  return amInfos;
}
Example 5
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
protected void serviceStart() throws Exception {

  amInfos = new LinkedList<AMInfo>();
  completedTasksFromPreviousRun = new HashMap<TaskId, TaskInfo>();
  processRecovery();

  // Current an AMInfo for the current AM generation.
  AMInfo amInfo =
      MRBuilderUtils.newAMInfo(appAttemptID, startTime, containerID, nmHost,
          nmPort, nmHttpPort);

  // /////////////////// Create the job itself.
  job = createJob(getConfig(), forcedState, shutDownMessage);

  // End of creating the job.

  // Send out an MR AM inited event for all previous AMs.
  for (AMInfo info : amInfos) {
    dispatcher.getEventHandler().handle(
        new JobHistoryEvent(job.getID(), new AMStartedEvent(info
            .getAppAttemptId(), info.getStartTime(), info.getContainerId(),
            info.getNodeManagerHost(), info.getNodeManagerPort(), info
                .getNodeManagerHttpPort(), appSubmitTime)));
  }

  // Send out an MR AM inited event for this AM.
  dispatcher.getEventHandler().handle(
      new JobHistoryEvent(job.getID(), new AMStartedEvent(amInfo
          .getAppAttemptId(), amInfo.getStartTime(), amInfo.getContainerId(),
          amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort(), amInfo
              .getNodeManagerHttpPort(), this.forcedState == null ? null
                  : this.forcedState.toString(), appSubmitTime)));
  amInfos.add(amInfo);

  // metrics system init is really init & start.
  // It's more test friendly to put it here.
  DefaultMetricsSystem.initialize("MRAppMaster");

  boolean initFailed = false;
  if (!errorHappenedShutDown) {
    // create a job event for job intialization
    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
    // Send init to the job (this does NOT trigger job execution)
    // This is a synchronous call, not an event through dispatcher. We want
    // job-init to be done completely here.
    jobEventDispatcher.handle(initJobEvent);

    // If job is still not initialized, an error happened during
    // initialization. Must complete starting all of the services so failure
    // events can be processed.
    initFailed = (((JobImpl)job).getInternalState() != JobStateInternal.INITED);

    // JobImpl's InitTransition is done (call above is synchronous), so the
    // "uber-decision" (MR-1220) has been made.  Query job and switch to
    // ubermode if appropriate (by registering different container-allocator
    // and container-launcher services/event-handlers).

    if (job.isUber()) {
      speculatorEventDispatcher.disableSpeculation();
      LOG.info("MRAppMaster uberizing job " + job.getID()
          + " in local container (\"uber-AM\") on node "
          + nmHost + ":" + nmPort + ".");
    } else {
      // send init to speculator only for non-uber jobs. 
      // This won't yet start as dispatcher isn't started yet.
      dispatcher.getEventHandler().handle(
          new SpeculatorEvent(job.getID(), clock.getTime()));
      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
          + "job " + job.getID() + ".");
    }
    // Start ClientService here, since it's not initialized if
    // errorHappenedShutDown is true
    clientService.start();
  }
  //start all the components
  super.serviceStart();

  // finally set the job classloader
  MRApps.setClassLoader(jobClassLoader, getConfig());

  if (initFailed) {
    JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
    jobEventDispatcher.handle(initFailedEvent);
  } else {
    // All components have started, start the job.
    startJobs();
  }
}
Example 6
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void parsePreviousJobHistory() throws IOException {
  FSDataInputStream in = getPreviousJobHistoryStream(getConfig(),
      appAttemptID);
  JobHistoryParser parser = new JobHistoryParser(in);
  JobInfo jobInfo = parser.parse();
  Exception parseException = parser.getParseException();
  if (parseException != null) {
    LOG.info("Got an error parsing job-history file" +
        ", ignoring incomplete events.", parseException);
  }
  Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
      .getAllTasks();
  for (TaskInfo taskInfo : taskInfos.values()) {
    if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
      Iterator<Entry<TaskAttemptID, TaskAttemptInfo>> taskAttemptIterator =
          taskInfo.getAllTaskAttempts().entrySet().iterator();
      while (taskAttemptIterator.hasNext()) {
        Map.Entry<TaskAttemptID, TaskAttemptInfo> currentEntry = taskAttemptIterator.next();
        if (!jobInfo.getAllCompletedTaskAttempts().containsKey(currentEntry.getKey())) {
          taskAttemptIterator.remove();
        }
      }
      completedTasksFromPreviousRun
          .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo);
      LOG.info("Read from history task "
          + TypeConverter.toYarn(taskInfo.getTaskId()));
    }
  }
  LOG.info("Read completed tasks from history "
      + completedTasksFromPreviousRun.size());
  recoveredJobStartTime = jobInfo.getLaunchTime();

  // recover AMInfos
  List<JobHistoryParser.AMInfo> jhAmInfoList = jobInfo.getAMInfos();
  if (jhAmInfoList != null) {
    for (JobHistoryParser.AMInfo jhAmInfo : jhAmInfoList) {
      AMInfo amInfo = MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
          jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
          jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
          jhAmInfo.getNodeManagerHttpPort());
      amInfos.add(amInfo);
    }
  }
}
Example 7
Source File: MRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
protected void serviceStart() throws Exception {

  amInfos = new LinkedList<AMInfo>();
  completedTasksFromPreviousRun = new HashMap<TaskId, TaskInfo>();
  processRecovery();

  // Current an AMInfo for the current AM generation.
  AMInfo amInfo =
      MRBuilderUtils.newAMInfo(appAttemptID, startTime, containerID, nmHost,
          nmPort, nmHttpPort);

  // /////////////////// Create the job itself.
  job = createJob(getConfig(), forcedState, shutDownMessage);

  // End of creating the job.

  // Send out an MR AM inited event for all previous AMs.
  for (AMInfo info : amInfos) {
    dispatcher.getEventHandler().handle(
        new JobHistoryEvent(job.getID(), new AMStartedEvent(info
            .getAppAttemptId(), info.getStartTime(), info.getContainerId(),
            info.getNodeManagerHost(), info.getNodeManagerPort(), info
                .getNodeManagerHttpPort(), appSubmitTime)));
  }

  // Send out an MR AM inited event for this AM.
  dispatcher.getEventHandler().handle(
      new JobHistoryEvent(job.getID(), new AMStartedEvent(amInfo
          .getAppAttemptId(), amInfo.getStartTime(), amInfo.getContainerId(),
          amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort(), amInfo
              .getNodeManagerHttpPort(), this.forcedState == null ? null
                  : this.forcedState.toString(), appSubmitTime)));
  amInfos.add(amInfo);

  // metrics system init is really init & start.
  // It's more test friendly to put it here.
  DefaultMetricsSystem.initialize("MRAppMaster");

  boolean initFailed = false;
  if (!errorHappenedShutDown) {
    // create a job event for job intialization
    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
    // Send init to the job (this does NOT trigger job execution)
    // This is a synchronous call, not an event through dispatcher. We want
    // job-init to be done completely here.
    jobEventDispatcher.handle(initJobEvent);

    // If job is still not initialized, an error happened during
    // initialization. Must complete starting all of the services so failure
    // events can be processed.
    initFailed = (((JobImpl)job).getInternalState() != JobStateInternal.INITED);

    // JobImpl's InitTransition is done (call above is synchronous), so the
    // "uber-decision" (MR-1220) has been made.  Query job and switch to
    // ubermode if appropriate (by registering different container-allocator
    // and container-launcher services/event-handlers).

    if (job.isUber()) {
      speculatorEventDispatcher.disableSpeculation();
      LOG.info("MRAppMaster uberizing job " + job.getID()
          + " in local container (\"uber-AM\") on node "
          + nmHost + ":" + nmPort + ".");
    } else {
      // send init to speculator only for non-uber jobs. 
      // This won't yet start as dispatcher isn't started yet.
      dispatcher.getEventHandler().handle(
          new SpeculatorEvent(job.getID(), clock.getTime()));
      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
          + "job " + job.getID() + ".");
    }
    // Start ClientService here, since it's not initialized if
    // errorHappenedShutDown is true
    clientService.start();
  }
  //start all the components
  super.serviceStart();

  // finally set the job classloader
  MRApps.setClassLoader(jobClassLoader, getConfig());

  if (initFailed) {
    JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
    jobEventDispatcher.handle(initFailedEvent);
  } else {
    // All components have started, start the job.
    startJobs();
  }
}
Example 8
Source File: MRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void parsePreviousJobHistory() throws IOException {
  FSDataInputStream in = getPreviousJobHistoryStream(getConfig(),
      appAttemptID);
  JobHistoryParser parser = new JobHistoryParser(in);
  JobInfo jobInfo = parser.parse();
  Exception parseException = parser.getParseException();
  if (parseException != null) {
    LOG.info("Got an error parsing job-history file" +
        ", ignoring incomplete events.", parseException);
  }
  Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
      .getAllTasks();
  for (TaskInfo taskInfo : taskInfos.values()) {
    if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
      Iterator<Entry<TaskAttemptID, TaskAttemptInfo>> taskAttemptIterator =
          taskInfo.getAllTaskAttempts().entrySet().iterator();
      while (taskAttemptIterator.hasNext()) {
        Map.Entry<TaskAttemptID, TaskAttemptInfo> currentEntry = taskAttemptIterator.next();
        if (!jobInfo.getAllCompletedTaskAttempts().containsKey(currentEntry.getKey())) {
          taskAttemptIterator.remove();
        }
      }
      completedTasksFromPreviousRun
          .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo);
      LOG.info("Read from history task "
          + TypeConverter.toYarn(taskInfo.getTaskId()));
    }
  }
  LOG.info("Read completed tasks from history "
      + completedTasksFromPreviousRun.size());
  recoveredJobStartTime = jobInfo.getLaunchTime();

  // recover AMInfos
  List<JobHistoryParser.AMInfo> jhAmInfoList = jobInfo.getAMInfos();
  if (jhAmInfoList != null) {
    for (JobHistoryParser.AMInfo jhAmInfo : jhAmInfoList) {
      AMInfo amInfo = MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
          jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
          jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
          jhAmInfo.getNodeManagerHttpPort());
      amInfos.add(amInfo);
    }
  }
}