Java Code Examples for org.apache.hadoop.mapreduce.v2.util.MRApps#taskType()

The following examples show how to use org.apache.hadoop.mapreduce.v2.util.MRApps#taskType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AMWebServices.java From big-c with Apache License 2.0

6 votes

@GET
@Path("/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}

Example 2

Source File: TestHsWebServicesTasks.java From big-c with Apache License 2.0

6 votes

public void verifyHsTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyHsSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}

Example 3

Source File: HsWebServices.java From big-c with Apache License 2.0

6 votes

@GET
@Path("/mapreduce/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}

Example 4

Source File: HsAttemptsPage.java From big-c with Apache License 2.0

6 votes

@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  Job j = app.getJob();
  Map<TaskId, Task> tasks = j.getTasks(taskType);
  for (Task task : tasks.values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}

Example 5

Source File: TestAMWebServicesTasks.java From big-c with Apache License 2.0

6 votes

public void verifyAMTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyAMSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}

Example 6

Source File: AttemptsPage.java From big-c with Apache License 2.0

6 votes

@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  for (Task task : super.app.getJob().getTasks(taskType).values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}

Example 7

Source File: AMWebServices.java From hadoop with Apache License 2.0

6 votes

@GET
@Path("/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}

Example 8

Source File: TestHsWebServicesTasks.java From hadoop with Apache License 2.0

6 votes

public void verifyHsTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyHsSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}

Example 9

Source File: HsWebServices.java From hadoop with Apache License 2.0

6 votes

@GET
@Path("/mapreduce/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}

Example 10

Source File: HsAttemptsPage.java From hadoop with Apache License 2.0

6 votes

@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  Job j = app.getJob();
  Map<TaskId, Task> tasks = j.getTasks(taskType);
  for (Task task : tasks.values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}

Example 11

Source File: TestAMWebServicesTasks.java From hadoop with Apache License 2.0

6 votes

public void verifyAMTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyAMSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}

Example 12

Source File: AttemptsPage.java From hadoop with Apache License 2.0

6 votes

@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  for (Task task : super.app.getJob().getTasks(taskType).values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}

Example 13

Source File: HsTasksPage.java From hadoop with Apache License 2.0

5 votes

/**
 * @return the end of the JS map that is the jquery datatable configuration
 * for the tasks table.
 */
private String tasksTableInit() {
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  }
  StringBuilder b = tableInit().
  append(", 'aaData': tasksTableData")
  .append(", bDeferRender: true")
  .append(", bProcessing: true")

  .append("\n, aoColumnDefs: [\n")
  .append("{'sType':'string', 'aTargets': [ 0 ]")
  .append(", 'mRender': parseHadoopID }")

  .append(", {'sType':'numeric', 'aTargets': [ 4")
  .append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7")
  .append(" ], 'mRender': renderHadoopElapsedTime }")

  .append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5")
  .append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6")
  .append(" ], 'mRender': renderHadoopDate }]")

  // Sort by id upon page load
  .append("\n, aaSorting: [[0, 'asc']]")
  .append("}");
  return b.toString();
}

Example 14

Source File: HsTaskPage.java From hadoop with Apache License 2.0

5 votes

/**
 * @return The end of the JS map that is the jquery datatable config for the
 * attempts table. 
 */
private String attemptsTableInit() {
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  } else {
    TaskId taskID = MRApps.toTaskID($(TASK_ID));
    type = taskID.getTaskType();
  }
  StringBuilder b = tableInit()
    .append(", 'aaData': attemptsTableData")
    .append(", bDeferRender: true")
    .append(", bProcessing: true")
    .append("\n,aoColumnDefs:[\n")

    //logs column should not filterable (it includes container ID which may pollute searches)
    .append("\n{'aTargets': [ 4 ]")
    .append(", 'bSearchable': false }")

    .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]")
    .append(", 'mRender': parseHadoopAttemptID }")

    .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
    //Column numbers are different for maps and reduces
    .append(type == TaskType.REDUCE ? ", 7, 8" : "")
    .append(" ], 'mRender': renderHadoopDate }")

    .append("\n, {'sType':'numeric', 'aTargets': [")
    .append(type == TaskType.REDUCE ? "9, 10, 11, 12" : "7")
    .append(" ], 'mRender': renderHadoopElapsedTime }]")

    // Sort by id upon page load
    .append("\n, aaSorting: [[0, 'asc']]")
    .append("}");
    return b.toString();
}

Example 15

Source File: HsTaskPage.java From big-c with Apache License 2.0

5 votes

/**
 * @return The end of the JS map that is the jquery datatable config for the
 * attempts table. 
 */
private String attemptsTableInit() {
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  } else {
    TaskId taskID = MRApps.toTaskID($(TASK_ID));
    type = taskID.getTaskType();
  }
  StringBuilder b = tableInit()
    .append(", 'aaData': attemptsTableData")
    .append(", bDeferRender: true")
    .append(", bProcessing: true")
    .append("\n,aoColumnDefs:[\n")

    //logs column should not filterable (it includes container ID which may pollute searches)
    .append("\n{'aTargets': [ 4 ]")
    .append(", 'bSearchable': false }")

    .append("\n, {'sType':'numeric', 'aTargets': [ 0 ]")
    .append(", 'mRender': parseHadoopAttemptID }")

    .append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
    //Column numbers are different for maps and reduces
    .append(type == TaskType.REDUCE ? ", 7, 8" : "")
    .append(" ], 'mRender': renderHadoopDate }")

    .append("\n, {'sType':'numeric', 'aTargets': [")
    .append(type == TaskType.REDUCE ? "9, 10, 11, 12" : "7")
    .append(" ], 'mRender': renderHadoopElapsedTime }]")

    // Sort by id upon page load
    .append("\n, aaSorting: [[0, 'asc']]")
    .append("}");
    return b.toString();
}

Example 16

Source File: HsTasksPage.java From big-c with Apache License 2.0

5 votes

/**
 * @return the end of the JS map that is the jquery datatable configuration
 * for the tasks table.
 */
private String tasksTableInit() {
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  }
  StringBuilder b = tableInit().
  append(", 'aaData': tasksTableData")
  .append(", bDeferRender: true")
  .append(", bProcessing: true")

  .append("\n, aoColumnDefs: [\n")
  .append("{'sType':'string', 'aTargets': [ 0 ]")
  .append(", 'mRender': parseHadoopID }")

  .append(", {'sType':'numeric', 'aTargets': [ 4")
  .append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7")
  .append(" ], 'mRender': renderHadoopElapsedTime }")

  .append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5")
  .append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6")
  .append(" ], 'mRender': renderHadoopDate }]")

  // Sort by id upon page load
  .append("\n, aaSorting: [[0, 'asc']]")
  .append("}");
  return b.toString();
}

Example 17

Source File: TasksBlock.java From big-c with Apache License 2.0

4 votes

@Override protected void render(Block html) {
  if (app.getJob() == null) {
    html.
      h2($(TITLE));
    return;
  }
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  }
  TBODY<TABLE<Hamlet>> tbody = html.
    table("#tasks").
      thead().
        tr().
          th("Task").
          th("Progress").
          th("Status").
          th("State").
          th("Start Time").
          th("Finish Time").
          th("Elapsed Time")._()._().
      tbody();
  StringBuilder tasksTableData = new StringBuilder("[\n");

  for (Task task : app.getJob().getTasks().values()) {
    if (type != null && task.getType() != type) {
      continue;
    }
    String taskStateStr = $(TASK_STATE);
    if (taskStateStr == null || taskStateStr.trim().equals("")) {
      taskStateStr = "ALL";
    }

    if (!taskStateStr.equalsIgnoreCase("ALL"))
    {
      try {
        // get stateUI enum
        MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr);
        if (!stateUI.correspondsTo(task.getState()))
        {
          continue;
        }
      } catch (IllegalArgumentException e) {
        continue; // not supported state, ignore
      }
    }

    TaskInfo info = new TaskInfo(task);
    String tid = info.getId();
    String pct = percent(info.getProgress() / 100);
    tasksTableData.append("[\"<a href='").append(url("task", tid))
    .append("'>").append(tid).append("</a>\",\"")
    //Progress bar
    .append("<br title='").append(pct)
    .append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
    .append(join(pct, '%')).append("'> ").append("<div class='")
    .append(C_PROGRESSBAR_VALUE).append("' style='")
    .append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
    .append(StringEscapeUtils.escapeJavaScript(
            StringEscapeUtils.escapeHtml(info.getStatus()))).append("\",\"")

    .append(info.getState()).append("\",\"")
    .append(info.getStartTime()).append("\",\"")
    .append(info.getFinishTime()).append("\",\"")
    .append(info.getElapsedTime()).append("\"],\n");
  }
  //Remove the last comma and close off the array of arrays
  if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
    tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1);
  }
  tasksTableData.append("]");
  html.script().$type("text/javascript").
  _("var tasksTableData=" + tasksTableData)._();

  tbody._()._();
}

Example 18

Source File: TasksBlock.java From hadoop with Apache License 2.0

4 votes

@Override protected void render(Block html) {
  if (app.getJob() == null) {
    html.
      h2($(TITLE));
    return;
  }
  TaskType type = null;
  String symbol = $(TASK_TYPE);
  if (!symbol.isEmpty()) {
    type = MRApps.taskType(symbol);
  }
  TBODY<TABLE<Hamlet>> tbody = html.
    table("#tasks").
      thead().
        tr().
          th("Task").
          th("Progress").
          th("Status").
          th("State").
          th("Start Time").
          th("Finish Time").
          th("Elapsed Time")._()._().
      tbody();
  StringBuilder tasksTableData = new StringBuilder("[\n");

  for (Task task : app.getJob().getTasks().values()) {
    if (type != null && task.getType() != type) {
      continue;
    }
    String taskStateStr = $(TASK_STATE);
    if (taskStateStr == null || taskStateStr.trim().equals("")) {
      taskStateStr = "ALL";
    }

    if (!taskStateStr.equalsIgnoreCase("ALL"))
    {
      try {
        // get stateUI enum
        MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr);
        if (!stateUI.correspondsTo(task.getState()))
        {
          continue;
        }
      } catch (IllegalArgumentException e) {
        continue; // not supported state, ignore
      }
    }

    TaskInfo info = new TaskInfo(task);
    String tid = info.getId();
    String pct = percent(info.getProgress() / 100);
    tasksTableData.append("[\"<a href='").append(url("task", tid))
    .append("'>").append(tid).append("</a>\",\"")
    //Progress bar
    .append("<br title='").append(pct)
    .append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
    .append(join(pct, '%')).append("'> ").append("<div class='")
    .append(C_PROGRESSBAR_VALUE).append("' style='")
    .append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
    .append(StringEscapeUtils.escapeJavaScript(
            StringEscapeUtils.escapeHtml(info.getStatus()))).append("\",\"")

    .append(info.getState()).append("\",\"")
    .append(info.getStartTime()).append("\",\"")
    .append(info.getFinishTime()).append("\",\"")
    .append(info.getElapsedTime()).append("\"],\n");
  }
  //Remove the last comma and close off the array of arrays
  if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
    tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1);
  }
  tasksTableData.append("]");
  html.script().$type("text/javascript").
  _("var tasksTableData=" + tasksTableData)._();

  tbody._()._();
}