Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.TaskType#REDUCE

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskType#REDUCE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AMWebServices.java From hadoop with Apache License 2.0

6 votes

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}

Example 2

Source File: AMWebServices.java From hadoop with Apache License 2.0

6 votes

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid,
    @PathParam("attemptid") String attId) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);
  TaskAttempt ta = getTaskAttemptFromTaskAttemptString(attId, task);
  if (task.getType() == TaskType.REDUCE) {
    return new ReduceTaskAttemptInfo(ta, task.getType());
  } else {
    return new TaskAttemptInfo(ta, task.getType(), true);
  }
}

Example 3

Source File: StartEndTimesBase.java From big-c with Apache License 2.0

6 votes

protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}

Example 4

Source File: TestAMWebServicesAttempts.java From big-c with Apache License 2.0

6 votes

public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}

Example 5

Source File: TestHsWebServicesAttempts.java From big-c with Apache License 2.0

6 votes

public void verifyHsTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}

Example 6

Source File: TestHsWebServicesAttempts.java From hadoop with Apache License 2.0

6 votes

public void verifyHsTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}

Example 7

Source File: StartEndTimesBase.java From hadoop with Apache License 2.0

6 votes

protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}

Example 8

Source File: TestHsWebServicesAttempts.java From big-c with Apache License 2.0

6 votes

public void verifyHsTaskAttempt(JSONObject info, TaskAttempt att,
    TaskType ttype) throws JSONException {
  if (ttype == TaskType.REDUCE) {
    assertEquals("incorrect number of elements", 17, info.length());
  } else {
    assertEquals("incorrect number of elements", 12, info.length());
  }

  verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
      info.getString("state"), info.getString("type"),
      info.getString("rack"), info.getString("nodeHttpAddress"),
      info.getString("diagnostics"), info.getString("assignedContainerId"),
      info.getLong("startTime"), info.getLong("finishTime"),
      info.getLong("elapsedTime"), (float) info.getDouble("progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
        info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
        info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
  }
}

Example 9

Source File: TestAMWebServicesAttempts.java From hadoop with Apache License 2.0

6 votes

public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}

Example 10

Source File: TestMRApps.java From hadoop with Apache License 2.0

5 votes

@Test (timeout = 120000)
public void testTaskIDtoString() {
  TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class);
  tid.setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class));
  tid.getJobId().setAppId(ApplicationId.newInstance(0, 0));
  tid.setTaskType(TaskType.MAP);
  TaskType type = tid.getTaskType();
  System.err.println(type);
  type = TaskType.REDUCE;
  System.err.println(type);
  System.err.println(tid.getTaskType());
  assertEquals("task_0_0000_m_000000", MRApps.toString(tid));
  tid.setTaskType(TaskType.REDUCE);
  assertEquals("task_0_0000_r_000000", MRApps.toString(tid));
}

Example 11

Source File: TypeConverter.java From hadoop with Apache License 2.0

5 votes

public static TaskType
    toYarn(org.apache.hadoop.mapreduce.TaskType taskType) {
  switch (taskType) {
  case MAP:
    return TaskType.MAP;
  case REDUCE:
    return TaskType.REDUCE;
  default:
    throw new YarnRuntimeException("Unrecognized task type: " + taskType);
  }
}

Example 12

Source File: MockJobs.java From hadoop with Apache License 2.0

5 votes

public static TaskAttemptReport newTaskAttemptReport(TaskAttemptId id) {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      id.getTaskId().getJobId().getAppId(), 0);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
  TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
  report.setTaskAttemptId(id);
  report
      .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
  report.setFinishTime(System.currentTimeMillis()
      + (int) (Math.random() * DT) + 1);

  if (id.getTaskId().getTaskType() == TaskType.REDUCE) {
    report.setShuffleFinishTime(
        (report.getFinishTime() + report.getStartTime()) / 2);
    report.setSortFinishTime(
        (report.getFinishTime() + report.getShuffleFinishTime()) / 2);
  }

  report.setPhase(PHASES.next());
  report.setTaskAttemptState(TASK_ATTEMPT_STATES.next());
  report.setProgress((float) Math.random());
  report.setCounters(TypeConverter.toYarn(newCounters()));
  report.setContainerId(containerId);
  report.setDiagnosticInfo(DIAGS.next());
  report.setStateString("Moving average " + Math.random());
  return report;
}

Example 13

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

5 votes

private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}

Example 14

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

5 votes

private int getCpuRequired(Configuration conf, TaskType taskType) {
  int vcores = 1;
  if (taskType == TaskType.MAP)  {
    vcores =
        conf.getInt(MRJobConfig.MAP_CPU_VCORES,
            MRJobConfig.DEFAULT_MAP_CPU_VCORES);
  } else if (taskType == TaskType.REDUCE) {
    vcores =
        conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
            MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
  }
  
  return vcores;
}

Example 15

Source File: JobImpl.java From hadoop with Apache License 2.0

5 votes

private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}

Example 16

Source File: TypeConverter.java From big-c with Apache License 2.0

5 votes

public static TaskType
    toYarn(org.apache.hadoop.mapreduce.TaskType taskType) {
  switch (taskType) {
  case MAP:
    return TaskType.MAP;
  case REDUCE:
    return TaskType.REDUCE;
  default:
    throw new YarnRuntimeException("Unrecognized task type: " + taskType);
  }
}

Example 17

Source File: ReduceTaskImpl.java From hadoop with Apache License 2.0

5 votes

public ReduceTaskImpl(JobId jobId, int partition,
    EventHandler eventHandler, Path jobFile, JobConf conf,
    int numMapTasks, TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.REDUCE, partition, eventHandler, jobFile, conf,
      taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.numMapTasks = numMapTasks;
}

Example 18

Source File: TestKill.java From big-c with Apache License 2.0

4 votes

@Test
public void testKillTaskWait() throws Exception {
  final Dispatcher dispatcher = new AsyncDispatcher() {
    private TaskAttemptEvent cachedKillEvent;
    @Override
    protected void dispatch(Event event) {
      if (event instanceof TaskAttemptEvent) {
        TaskAttemptEvent killEvent = (TaskAttemptEvent) event;
        if (killEvent.getType() == TaskAttemptEventType.TA_KILL) {
          TaskAttemptId taID = killEvent.getTaskAttemptID();
          if (taID.getTaskId().getTaskType() == TaskType.REDUCE
              && taID.getTaskId().getId() == 0 && taID.getId() == 0) {
            // Task is asking the reduce TA to kill itself. 'Create' a race
            // condition. Make the task succeed and then inform the task that
            // TA has succeeded. Once Task gets the TA succeeded event at
            // KILL_WAIT, then relay the actual kill signal to TA
            super.dispatch(new TaskAttemptEvent(taID,
              TaskAttemptEventType.TA_DONE));
            super.dispatch(new TaskAttemptEvent(taID,
              TaskAttemptEventType.TA_CONTAINER_CLEANED));
            super.dispatch(new TaskTAttemptEvent(taID,
              TaskEventType.T_ATTEMPT_SUCCEEDED));
            this.cachedKillEvent = killEvent;
            return;
          }
        }
      } else if (event instanceof TaskEvent) {
        TaskEvent taskEvent = (TaskEvent) event;
        if (taskEvent.getType() == TaskEventType.T_ATTEMPT_SUCCEEDED
            && this.cachedKillEvent != null) {
          // When the TA comes and reports that it is done, send the
          // cachedKillEvent
          super.dispatch(this.cachedKillEvent);
          return;
        }

      }
      super.dispatch(event);
    }
  };
  MRApp app = new MRApp(1, 1, false, this.getClass().getName(), true) {
    @Override
    public Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  Job job = app.submit(new Configuration());
  JobId jobId = app.getJobId();
  app.waitForState(job, JobState.RUNNING);
  Assert.assertEquals("Num tasks not correct", 2, job.getTasks().size());
  Iterator<Task> it = job.getTasks().values().iterator();
  Task mapTask = it.next();
  Task reduceTask = it.next();
  app.waitForState(mapTask, TaskState.RUNNING);
  app.waitForState(reduceTask, TaskState.RUNNING);
  TaskAttempt mapAttempt = mapTask.getAttempts().values().iterator().next();
  app.waitForState(mapAttempt, TaskAttemptState.RUNNING);
  TaskAttempt reduceAttempt = reduceTask.getAttempts().values().iterator().next();
  app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);

  // Finish map
  app.getContext().getEventHandler().handle(
      new TaskAttemptEvent(
          mapAttempt.getID(),
          TaskAttemptEventType.TA_DONE));
  app.waitForState(mapTask, TaskState.SUCCEEDED);

  // Now kill the job
  app.getContext().getEventHandler()
    .handle(new JobEvent(jobId, JobEventType.JOB_KILL));

  app.waitForInternalState((JobImpl)job, JobStateInternal.KILLED);
}

Example 19

Source File: ReduceTaskImpl.java From hadoop with Apache License 2.0

4 votes

@Override
public TaskType getType() {
  return TaskType.REDUCE;
}

Example 20

Source File: MRApps.java From hadoop with Apache License 2.0

4 votes

public static TaskType taskType(String symbol) {
  // JDK 7 supports switch on strings
  if (symbol.equals("m")) return TaskType.MAP;
  if (symbol.equals("r")) return TaskType.REDUCE;
  throw new YarnRuntimeException("Unknown task symbol: "+ symbol);
}