org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LegacyTaskRuntimeEstimator.java    From big-c with Apache License 2.0 6 votes vote down vote up
private long storedPerAttemptValue
     (Map<TaskAttempt, AtomicLong> data, TaskAttemptId attemptID) {
  TaskId taskID = attemptID.getTaskId();
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  Task task = job.getTask(taskID);

  if (task == null) {
    return -1L;
  }

  TaskAttempt taskAttempt = task.getAttempt(attemptID);

  if (taskAttempt == null) {
    return -1L;
  }

  AtomicLong estimate = data.get(taskAttempt);

  return estimate == null ? -1L : estimate.get();

}
 
Example #2
Source File: TestTaskAttempt.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void testMRAppHistory(MRApp app) throws Exception {
  Configuration conf = new Configuration();
  Job job = app.submit(conf);
  app.waitForState(job, JobState.FAILED);
  Map<TaskId, Task> tasks = job.getTasks();

  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
  Task task = tasks.values().iterator().next();
  Assert.assertEquals("Task state not correct", TaskState.FAILED, task
      .getReport().getTaskState());
  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
      .getAttempts();
  Assert.assertEquals("Num attempts is not correct", 4, attempts.size());

  Iterator<TaskAttempt> it = attempts.values().iterator();
  TaskAttemptReport report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
  Assert.assertEquals("Diagnostic Information is not Correct",
      "Test Diagnostic Event", report.getDiagnosticInfo());
  report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
}
 
Example #3
Source File: TestAMWebServicesAttempts.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void verifyAMTaskAttemptsXML(NodeList nodes, Task task) {
  assertEquals("incorrect number of elements", 1, nodes.getLength());

  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;
    for (int i = 0; i < nodes.getLength(); i++) {
      Element element = (Element) nodes.item(i);

      if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
        found = true;
        verifyAMTaskAttemptXML(element, att, task.getType());
      }
    }
    assertTrue("task with id: " + attid + " not in web service output", found);
  }
}
 
Example #4
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}
 
Example #5
Source File: TestAMWebServicesAttempts.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void verifyAMTaskAttemptsXML(NodeList nodes, Task task) {
  assertEquals("incorrect number of elements", 1, nodes.getLength());

  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;
    for (int i = 0; i < nodes.getLength(); i++) {
      Element element = (Element) nodes.item(i);

      if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
        found = true;
        verifyAMTaskAttemptXML(element, att, task.getType());
      }
    }
    assertTrue("task with id: " + attid + " not in web service output", found);
  }
}
 
Example #6
Source File: TestTaskAttempt.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void testMRAppHistory(MRApp app) throws Exception {
  Configuration conf = new Configuration();
  Job job = app.submit(conf);
  app.waitForState(job, JobState.FAILED);
  Map<TaskId, Task> tasks = job.getTasks();

  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
  Task task = tasks.values().iterator().next();
  Assert.assertEquals("Task state not correct", TaskState.FAILED, task
      .getReport().getTaskState());
  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
      .getAttempts();
  Assert.assertEquals("Num attempts is not correct", 4, attempts.size());

  Iterator<TaskAttempt> it = attempts.values().iterator();
  TaskAttemptReport report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
  Assert.assertEquals("Diagnostic Information is not Correct",
      "Test Diagnostic Event", report.getDiagnosticInfo());
  report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
}
 
Example #7
Source File: TestFetchFailure.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
  TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
  status.counters = new Counters();
  status.fetchFailedMaps = new ArrayList<TaskAttemptId>();
  status.id = attempt.getID();
  status.mapFinishTime = 0;
  status.phase = phase;
  status.progress = 0.5f;
  status.shuffleFinishTime = 0;
  status.sortFinishTime = 0;
  status.stateString = "OK";
  status.taskState = attempt.getState();
  TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(attempt.getID(),
      status);
  app.getContext().getEventHandler().handle(event);
}
 
Example #8
Source File: RMContainerAllocator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
boolean remove(TaskAttemptId tId) {
  ContainerRequest req = null;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    req = maps.remove(tId);
  } else {
    req = reduces.remove(tId);
  }
  
  if (req == null) {
    return false;
  } else {
    decContainerReq(req);
    return true;
  }
}
 
Example #9
Source File: TaskReportPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void addAllRunningAttempts(final List<TaskAttemptId> runningAttempts) {
  if (runningAttempts == null)
    return;
  initRunningAttempts();
  this.runningAttempts.addAll(runningAttempts);
}
 
Example #10
Source File: TaskHeartbeatHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock,
    int numThreads) {
  super("TaskHeartbeatHandler");
  this.eventHandler = eventHandler;
  this.clock = clock;
  runningAttempts =
    new ConcurrentHashMap<TaskAttemptId, ReportTime>(16, 0.75f, numThreads);
}
 
Example #11
Source File: TestSpeculativeExecution.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public long estimatedRuntime(TaskAttemptId id) {
  if ((id.getTaskId().getId() == 0) && (id.getId() == 0)) {
    return SPECULATE_THIS;
  }
  return super.estimatedRuntime(id);
}
 
Example #12
Source File: NotRunningJob.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public GetTaskReportResponse getTaskReport(GetTaskReportRequest request)
    throws IOException {
  GetTaskReportResponse resp =
    recordFactory.newRecordInstance(GetTaskReportResponse.class);
  TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
  report.setTaskId(request.getTaskId());
  report.setTaskState(TaskState.NEW);
  Counters counters = recordFactory.newRecordInstance(Counters.class);
  counters.addAllCounterGroups(new HashMap<String, CounterGroup>());
  report.setCounters(counters);
  report.addAllRunningAttempts(new ArrayList<TaskAttemptId>());
  return resp;
}
 
Example #13
Source File: TaskAttemptCompletionEventPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public TaskAttemptId getAttemptId() {
  TaskAttemptCompletionEventProtoOrBuilder p = viaProto ? proto : builder;
  if (this.taskAttemptId != null) {
    return this.taskAttemptId;
  }
  if (!p.hasAttemptId()) {
    return null;
  }
  this.taskAttemptId = convertFromProtoFormat(p.getAttemptId());
  return this.taskAttemptId;
}
 
Example #14
Source File: TaskHeartbeatHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void progressing(TaskAttemptId attemptID) {
//only put for the registered attempts
  //TODO throw an exception if the task isn't registered.
  ReportTime time = runningAttempts.get(attemptID);
  if(time != null) {
    time.setLastProgress(clock.getTime());
  }
}
 
Example #15
Source File: TaskAttemptCompletionEventPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public TaskAttemptId getAttemptId() {
  TaskAttemptCompletionEventProtoOrBuilder p = viaProto ? proto : builder;
  if (this.taskAttemptId != null) {
    return this.taskAttemptId;
  }
  if (!p.hasAttemptId()) {
    return null;
  }
  this.taskAttemptId = convertFromProtoFormat(p.getAttemptId());
  return this.taskAttemptId;
}
 
Example #16
Source File: TaskAttemptCompletionEventPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void setAttemptId(TaskAttemptId attemptId) {
  maybeInitBuilder();
  if (attemptId == null) 
    builder.clearAttemptId();
  this.taskAttemptId = attemptId;
}
 
Example #17
Source File: RMContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up
ContainerId get(TaskAttemptId tId) {
  Container taskContainer;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    taskContainer = maps.get(tId);
  } else {
    taskContainer = reduces.get(tId);
  }

  if (taskContainer == null) {
    return null;
  } else {
    return taskContainer.getId();
  }
}
 
Example #18
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}
 
Example #19
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskAttemptIdCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("history")
            .path("mapreduce").path("jobs").path(jobId).path("tasks")
            .path(tid).path("attempts").path(attid).path("counters")
            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("jobTaskAttemptCounters");
        verifyHsJobTaskAttemptCounters(info, att);
      }
    }
  }
}
 
Example #20
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ContainerFailedEvent createFailEvent(JobId jobId, int taskAttemptId,
    String host, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  return new ContainerFailedEvent(attemptId, host);    
}
 
Example #21
Source File: TaskAttemptRecoverEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public TaskAttemptRecoverEvent(TaskAttemptId id, TaskAttemptInfo taInfo,
    OutputCommitter committer, boolean recoverOutput) {
  super(id, TaskAttemptEventType.TA_RECOVER);
  this.taInfo = taInfo;
  this.committer = committer;
  this.recoverAttemptOutput = recoverOutput;
}
 
Example #22
Source File: MRApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void attemptLaunched(TaskAttemptId attemptID) {
  if (autoComplete) {
    // send the done event
    getContext().getEventHandler().handle(
        new TaskAttemptEvent(attemptID,
            TaskAttemptEventType.TA_DONE));
  }
}
 
Example #23
Source File: LocalContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Within the _local_ filesystem (not HDFS), all activity takes place within
 * a subdir inside one of the LOCAL_DIRS
 * (${local.dir}/usercache/$user/appcache/$appId/$contId/),
 * and all sub-MapTasks create the same filename ("file.out").  Rename that
 * to something unique (e.g., "map_0.out") to avoid possible collisions.
 *
 * Longer-term, we'll modify [something] to use TaskAttemptID-based
 * filenames instead of "file.out". (All of this is entirely internal,
 * so there are no particular compatibility issues.)
 */
@VisibleForTesting
protected static MapOutputFile renameMapOutputForReduce(JobConf conf,
    TaskAttemptId mapId, MapOutputFile subMapOutputFile) throws IOException {
  FileSystem localFs = FileSystem.getLocal(conf);
  // move map output to reduce input
  Path mapOut = subMapOutputFile.getOutputFile();
  FileStatus mStatus = localFs.getFileStatus(mapOut);
  Path reduceIn = subMapOutputFile.getInputFileForWrite(
      TypeConverter.fromYarn(mapId).getTaskID(), mStatus.getLen());
  Path mapOutIndex = subMapOutputFile.getOutputIndexFile();
  Path reduceInIndex = new Path(reduceIn.toString() + ".index");
  if (LOG.isDebugEnabled()) {
    LOG.debug("Renaming map output file for task attempt "
        + mapId.toString() + " from original location " + mapOut.toString()
        + " to destination " + reduceIn.toString());
  }
  if (!localFs.mkdirs(reduceIn.getParent())) {
    throw new IOException("Mkdirs failed to create "
        + reduceIn.getParent().toString());
  }
  if (!localFs.rename(mapOut, reduceIn))
    throw new IOException("Couldn't rename " + mapOut);
  if (!localFs.rename(mapOutIndex, reduceInIndex))
    throw new IOException("Couldn't rename " + mapOutIndex);

  return new RenamedMapOutputFile(reduceIn);
}
 
Example #24
Source File: TestAMWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskAttemptId() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("mapreduce")
            .path("jobs").path(jobId).path("tasks").path(tid)
            .path("attempts").path(attid).accept(MediaType.APPLICATION_JSON)
            .get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("taskAttempt");
        verifyAMTaskAttempt(info, att, task.getType());
      }
    }
  }
}
 
Example #25
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskAttemptIdDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("history")
            .path("mapreduce").path("jobs").path(jobId).path("tasks")
            .path(tid).path("attempts").path(attid).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("taskAttempt");
        verifyHsTaskAttempt(info, att, task.getType());
      }
    }
  }
}
 
Example #26
Source File: TypeConverter.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static TaskAttemptId toYarn(
    org.apache.hadoop.mapred.TaskAttemptID id) {
  TaskAttemptId taskAttemptId = recordFactory.newRecordInstance(TaskAttemptId.class);
  taskAttemptId.setTaskId(toYarn(id.getTaskID()));
  taskAttemptId.setId(id.getId());
  return taskAttemptId;
}
 
Example #27
Source File: TestAMWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskAttemptIdXMLCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("mapreduce")
            .path("jobs").path(jobId).path("tasks").path(tid)
            .path("attempts").path(attid).path("counters")
            .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);

        assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
        String xml = response.getEntity(String.class);
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        DocumentBuilder db = dbf.newDocumentBuilder();
        InputSource is = new InputSource();
        is.setCharacterStream(new StringReader(xml));
        Document dom = db.parse(is);
        NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters");

        verifyAMTaskCountersXML(nodes, att);
      }
    }
  }
}
 
Example #28
Source File: TaskReportPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public TaskAttemptId getSuccessfulAttempt() {
  TaskReportProtoOrBuilder p = viaProto ? proto : builder;
  if (this.successfulAttemptId != null) {
    return this.successfulAttemptId;
  }
  if (!p.hasSuccessfulAttempt()) {
    return null;
  }
  this.successfulAttemptId = convertFromProtoFormat(p.getSuccessfulAttempt());
  return this.successfulAttemptId;
}
 
Example #29
Source File: NotRunningJob.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public GetTaskReportResponse getTaskReport(GetTaskReportRequest request)
    throws IOException {
  GetTaskReportResponse resp =
    recordFactory.newRecordInstance(GetTaskReportResponse.class);
  TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
  report.setTaskId(request.getTaskId());
  report.setTaskState(TaskState.NEW);
  Counters counters = recordFactory.newRecordInstance(Counters.class);
  counters.addAllCounterGroups(new HashMap<String, CounterGroup>());
  report.setCounters(counters);
  report.addAllRunningAttempts(new ArrayList<TaskAttemptId>());
  return resp;
}
 
Example #30
Source File: HistoryClientService.java    From XLearning with Apache License 2.0 5 votes vote down vote up
@Override
public GetTaskAttemptReportResponse getTaskAttemptReport(
    GetTaskAttemptReportRequest request) throws IOException {
  TaskAttemptId taskAttemptId = request.getTaskAttemptId();
  Job job = verifyAndGetJob(taskAttemptId.getTaskId().getJobId(), true);
  GetTaskAttemptReportResponse response = recordFactory.newRecordInstance(GetTaskAttemptReportResponse.class);
  response.setTaskAttemptReport(job.getTask(taskAttemptId.getTaskId()).getAttempt(taskAttemptId).getReport());
  return response;
}