Java Code Examples for org.apache.hadoop.mapreduce.v2.app.job.Job

The following examples show how to use org.apache.hadoop.mapreduce.v2.app.job.Job. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestMRApp.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobRebootOnLastRetryOnUnregistrationFailure()
    throws Exception {
  // make startCount as 2 since this is last retry which equals to
  // DEFAULT_MAX_AM_RETRY
  // The last param mocks the unregistration failure
  MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true, 2, false);

  Configuration conf = new Configuration();
  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Assert.assertEquals("Num tasks not correct", 1, job.getTasks().size());
  Iterator<Task> it = job.getTasks().values().iterator();
  Task task = it.next();
  app.waitForState(task, TaskState.RUNNING);

  //send an reboot event
  app.getContext().getEventHandler().handle(new JobEvent(job.getID(),
    JobEventType.JOB_AM_REBOOT));

  app.waitForInternalState((JobImpl) job, JobStateInternal.REBOOT);
  // return exteranl state as RUNNING if this is the last retry while
  // unregistration fails
  app.waitForState(job, JobState.RUNNING);
}
 
Example 2
Source Project: big-c   Source File: TestHsWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskIdCountersDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters").get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyHsJobTaskCounters(info, task);
    }
  }
}
 
Example 3
Source Project: big-c   Source File: TestHsWebServicesJobsQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobsQueryState() throws JSONException, Exception {
  WebResource r = resource();
  // we only create 3 jobs and it cycles through states so we should have 3 unique states
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  String queryState = "BOGUS";
  JobId jid = null;
  for (Map.Entry<JobId, Job> entry : jobsMap.entrySet()) {
    jid = entry.getValue().getID();
    queryState = entry.getValue().getState().toString();
    break;
  }
  ClientResponse response = r.path("ws").path("v1").path("history")
      .path("mapreduce").path("jobs").queryParam("state", queryState)
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject jobs = json.getJSONObject("jobs");
  JSONArray arr = jobs.getJSONArray("job");
  assertEquals("incorrect number of elements", 1, arr.length());
  JSONObject info = arr.getJSONObject(0);
  Job job = appContext.getPartialJob(jid);
  VerifyJobsUtils.verifyHsJobPartial(info, job);
}
 
Example 4
Source Project: big-c   Source File: TestHsWebServicesJobConf.java    License: Apache License 2.0 6 votes vote down vote up
public void verifyHsJobConfXML(NodeList nodes, Job job) {

    assertEquals("incorrect number of elements", 1, nodes.getLength());

    for (int i = 0; i < nodes.getLength(); i++) {
      Element element = (Element) nodes.item(i);
      WebServicesTestUtils.checkStringMatch("path", job.getConfFile()
          .toString(), WebServicesTestUtils.getXmlString(element, "path"));

      // just do simple verification of fields - not data is correct
      // in the fields
      NodeList properties = element.getElementsByTagName("property");

      for (int j = 0; j < properties.getLength(); j++) {
        Element property = (Element) properties.item(j);
        assertNotNull("should have counters in the web service info", property);
        String name = WebServicesTestUtils.getXmlString(property, "name");
        String value = WebServicesTestUtils.getXmlString(property, "value");
        assertTrue("name not set", (name != null && !name.isEmpty()));
        assertTrue("name not set", (value != null && !value.isEmpty()));
      }
    }
  }
 
Example 5
Source Project: hadoop   Source File: MRApp.java    License: Apache License 2.0 6 votes vote down vote up
public void verifyCompleted() {
  for (Job job : getContext().getAllJobs().values()) {
    JobReport jobReport = job.getReport();
    System.out.println("Job start time :" + jobReport.getStartTime());
    System.out.println("Job finish time :" + jobReport.getFinishTime());
    Assert.assertTrue("Job start time is not less than finish time",
        jobReport.getStartTime() <= jobReport.getFinishTime());
    Assert.assertTrue("Job finish time is in future",
        jobReport.getFinishTime() <= System.currentTimeMillis());
    for (Task task : job.getTasks().values()) {
      TaskReport taskReport = task.getReport();
      System.out.println("Task start time : " + taskReport.getStartTime());
      System.out.println("Task finish time : " + taskReport.getFinishTime());
      Assert.assertTrue("Task start time is not less than finish time",
          taskReport.getStartTime() <= taskReport.getFinishTime());
      for (TaskAttempt attempt : task.getAttempts().values()) {
        TaskAttemptReport attemptReport = attempt.getReport();
        Assert.assertTrue("Attempt start time is not less than finish time",
            attemptReport.getStartTime() <= attemptReport.getFinishTime());
      }
    }
  }
}
 
Example 6
Source Project: big-c   Source File: HsAttemptsPage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Collection<TaskAttempt> getTaskAttempts() {
  List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>();
  String taskTypeStr = $(TASK_TYPE);
  TaskType taskType = MRApps.taskType(taskTypeStr);
  String attemptStateStr = $(ATTEMPT_STATE);
  TaskAttemptStateUI neededState = MRApps
      .taskAttemptState(attemptStateStr);
  Job j = app.getJob();
  Map<TaskId, Task> tasks = j.getTasks(taskType);
  for (Task task : tasks.values()) {
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    for (TaskAttempt attempt : attempts.values()) {
      if (neededState.correspondsTo(attempt.getState())) {
        fewTaskAttemps.add(attempt);
      }
    }
  }
  return fewTaskAttemps;
}
 
Example 7
Source Project: hadoop   Source File: TestAMWebServicesJobConf.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobConfXML() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("conf")
        .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
    String xml = response.getEntity(String.class);
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    InputSource is = new InputSource();
    is.setCharacterStream(new StringReader(xml));
    Document dom = db.parse(is);
    NodeList info = dom.getElementsByTagName("conf");
    verifyAMJobConfXML(info, jobsMap.get(id));
  }
}
 
Example 8
Source Project: hadoop   Source File: TestAMWebServicesAttempts.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskAttemptsDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid).path("attempts")
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      verifyAMTaskAttempts(json, task);
    }
  }
}
 
Example 9
Source Project: hadoop   Source File: AppController.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that a JOB_ID was passed into the page.
 */
public void requireJob() {
  if ($(JOB_ID).isEmpty()) {
    badRequest("missing job ID");
    throw new RuntimeException("Bad Request: Missing job ID");
  }

  JobId jobID = MRApps.toJobID($(JOB_ID));
  app.setJob(app.context.getJob(jobID));
  if (app.getJob() == null) {
    notFound($(JOB_ID));
    throw new RuntimeException("Not Found: " + $(JOB_ID));
  }

  /* check for acl access */
  Job job = app.context.getJob(jobID);
  if (!checkAccess(job)) {
    accessDenied("User " + request().getRemoteUser() + " does not have " +
        " permission to view job " + $(JOB_ID));
    throw new RuntimeException("Access denied: User " +
        request().getRemoteUser() + " does not have permission to view job " +
        $(JOB_ID));
  }
}
 
Example 10
Source Project: big-c   Source File: TestRuntimeEstimators.java    License: Apache License 2.0 6 votes vote down vote up
private float getReduceProgress() {
  Job job = myAppContext.getJob(myAttemptID.getTaskId().getJobId());
  float runtime = getCodeRuntime();

  Collection<Task> allMapTasks = job.getTasks(TaskType.MAP).values();

  int numberMaps = allMapTasks.size();
  int numberDoneMaps = 0;

  for (Task mapTask : allMapTasks) {
    if (mapTask.isFinished()) {
      ++numberDoneMaps;
    }
  }

  if (numberMaps == numberDoneMaps) {
    shuffleCompletedTime = Math.min(shuffleCompletedTime, clock.getTime());

    return Math.min
        ((float) (clock.getTime() - shuffleCompletedTime)
                    / (runtime * 2000.0F) + 0.5F,
         1.0F);
  } else {
    return ((float) numberDoneMaps) / numberMaps * 0.5F;
  }
}
 
Example 11
Source Project: big-c   Source File: TestStagingCleanup.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Job createJob(Configuration conf, JobStateInternal forcedState, 
    String diagnostic) {
  UserGroupInformation currentUser = null;
  try {
    currentUser = UserGroupInformation.getCurrentUser();
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  Job newJob = new TestJob(getJobId(), getAttemptID(), conf,
      getDispatcher().getEventHandler(),
      getTaskAttemptListener(), getContext().getClock(),
      getCommitter(), isNewApiCommitter(),
      currentUser.getUserName(), getContext(),
      forcedState, diagnostic);
  ((AppContext) getContext()).getAllJobs().put(newJob.getID(), newJob);

  getDispatcher().register(JobFinishEvent.Type.class,
      createJobFinishEventHandler());

  return newJob;
}
 
Example 12
Source Project: hadoop   Source File: TestBlocks.java    License: Apache License 2.0 6 votes vote down vote up
private Job getJob() {
  Job job = mock(Job.class);

  JobId jobId = new JobIdPBImpl();

  ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
  jobId.setAppId(appId);
  jobId.setId(1);
  when(job.getID()).thenReturn(jobId);

  JobReport report = mock(JobReport.class);
  when(report.getStartTime()).thenReturn(100010L);
  when(report.getFinishTime()).thenReturn(100015L);

  when(job.getReport()).thenReturn(report);
  when(job.getName()).thenReturn("JobName");
  when(job.getUserName()).thenReturn("UserName");
  when(job.getQueueName()).thenReturn("QueueName");
  when(job.getState()).thenReturn(JobState.SUCCEEDED);
  when(job.getTotalMaps()).thenReturn(3);
  when(job.getCompletedMaps()).thenReturn(2);
  when(job.getTotalReduces()).thenReturn(2);
  when(job.getCompletedReduces()).thenReturn(1);
  when(job.getCompletedReduces()).thenReturn(1);
  return job;
}
 
Example 13
Source Project: hadoop   Source File: AMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path("/jobs")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobsInfo getJobs(@Context HttpServletRequest hsr) {
  init();
  JobsInfo allJobs = new JobsInfo();
  for (Job job : appCtx.getAllJobs().values()) {
    // getAllJobs only gives you a partial we want a full
    Job fullJob = appCtx.getJob(job.getID());
    if (fullJob == null) {
      continue;
    }
    allJobs.add(new JobInfo(fullJob, hasAccess(fullJob, hsr)));
  }
  return allJobs;
}
 
Example 14
Source Project: big-c   Source File: TestAMWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobIdXML() throws Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).accept(MediaType.APPLICATION_XML)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
    String xml = response.getEntity(String.class);
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    InputSource is = new InputSource();
    is.setCharacterStream(new StringReader(xml));
    Document dom = db.parse(is);
    NodeList job = dom.getElementsByTagName("job");
    verifyAMJobXML(job, appContext);
  }

}
 
Example 15
Source Project: big-c   Source File: LegacyTaskRuntimeEstimator.java    License: Apache License 2.0 6 votes vote down vote up
private long storedPerAttemptValue
     (Map<TaskAttempt, AtomicLong> data, TaskAttemptId attemptID) {
  TaskId taskID = attemptID.getTaskId();
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  Task task = job.getTask(taskID);

  if (task == null) {
    return -1L;
  }

  TaskAttempt taskAttempt = task.getAttempt(attemptID);

  if (taskAttempt == null) {
    return -1L;
  }

  AtomicLong estimate = data.get(taskAttempt);

  return estimate == null ? -1L : estimate.get();

}
 
Example 16
Source Project: big-c   Source File: TestAMWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTasks() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("tasks")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject tasks = json.getJSONObject("tasks");
    JSONArray arr = tasks.getJSONArray("task");
    assertEquals("incorrect number of elements", 2, arr.length());

    verifyAMTask(arr, jobsMap.get(id), null);
  }
}
 
Example 17
Source Project: hadoop   Source File: TestHsWebServicesJobConf.java    License: Apache License 2.0 6 votes vote down vote up
public void verifyHsJobConf(JSONObject info, Job job) throws JSONException {

    assertEquals("incorrect number of elements", 2, info.length());

    WebServicesTestUtils.checkStringMatch("path", job.getConfFile().toString(),
        info.getString("path"));
    // just do simple verification of fields - not data is correct
    // in the fields
    JSONArray properties = info.getJSONArray("property");
    for (int i = 0; i < properties.length(); i++) {
      JSONObject prop = properties.getJSONObject(i);
      String name = prop.getString("name");
      String value = prop.getString("value");
      assertTrue("name not set", (name != null && !name.isEmpty()));
      assertTrue("value not set", (value != null && !value.isEmpty()));
    }
  }
 
Example 18
Source Project: big-c   Source File: TestAMWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobAttemptsDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1")
        .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobAttempts");
    verifyJobAttempts(info, jobsMap.get(id));
  }
}
 
Example 19
Source Project: hadoop   Source File: TestAMWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskIdCountersDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid).path("counters")
          .get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyAMJobTaskCounters(info, task);
    }
  }
}
 
Example 20
Source Project: big-c   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobAttemptsXML() throws Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
        .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
    String xml = response.getEntity(String.class);
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    InputSource is = new InputSource();
    is.setCharacterStream(new StringReader(xml));
    Document dom = db.parse(is);
    NodeList attempts = dom.getElementsByTagName("jobAttempts");
    assertEquals("incorrect number of elements", 1, attempts.getLength());
    NodeList info = dom.getElementsByTagName("jobAttempt");
    verifyHsJobAttemptsXML(info, appContext.getJob(id));
  }
}
 
Example 21
Source Project: hadoop   Source File: TestHsWebServicesTasks.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskIdCountersDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .path("counters").get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("jobTaskCounters");
      verifyHsJobTaskCounters(info, task);
    }
  }
}
 
Example 22
Source Project: big-c   Source File: TestAMWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobCountersXML() throws Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("counters")
        .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
    String xml = response.getEntity(String.class);
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    InputSource is = new InputSource();
    is.setCharacterStream(new StringReader(xml));
    Document dom = db.parse(is);
    NodeList info = dom.getElementsByTagName("jobCounters");
    verifyAMJobCountersXML(info, jobsMap.get(id));
  }
}
 
Example 23
Source Project: big-c   Source File: TestHsWebServicesJobs.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobAttempts() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobAttempts");
    verifyHsJobAttempts(info, appContext.getJob(id));
  }
}
 
Example 24
Source Project: hadoop   Source File: TestStagingCleanup.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Job createJob(Configuration conf, JobStateInternal forcedState, 
    String diagnostic) {
  UserGroupInformation currentUser = null;
  try {
    currentUser = UserGroupInformation.getCurrentUser();
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  Job newJob = new TestJob(getJobId(), getAttemptID(), conf,
      getDispatcher().getEventHandler(),
      getTaskAttemptListener(), getContext().getClock(),
      getCommitter(), isNewApiCommitter(),
      currentUser.getUserName(), getContext(),
      forcedState, diagnostic);
  ((AppContext) getContext()).getAllJobs().put(newJob.getID(), newJob);

  getDispatcher().register(JobFinishEvent.Type.class,
      createJobFinishEventHandler());

  return newJob;
}
 
Example 25
Source Project: hadoop   Source File: TestMRApp.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobError() throws Exception {
  MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true);
  Job job = app.submit(new Configuration());
  app.waitForState(job, JobState.RUNNING);
  Assert.assertEquals("Num tasks not correct", 1, job.getTasks().size());
  Iterator<Task> it = job.getTasks().values().iterator();
  Task task = it.next();
  app.waitForState(task, TaskState.RUNNING);

  //send an invalid event on task at current state
  app.getContext().getEventHandler().handle(
      new TaskEvent(
          task.getID(), TaskEventType.T_SCHEDULE));

  //this must lead to job error
  app.waitForState(job, JobState.ERROR);
}
 
Example 26
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example 27
Source Project: XLearning   Source File: HistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetCountersResponse getCounters(GetCountersRequest request)
    throws IOException {
  JobId jobId = request.getJobId();
  Job job = verifyAndGetJob(jobId, true);
  GetCountersResponse response = recordFactory.newRecordInstance(GetCountersResponse.class);
  response.setCounters(TypeConverter.toYarn(job.getAllCounters()));
  return response;
}
 
Example 28
Source Project: big-c   Source File: TestAMWebServicesAttempts.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskAttemptIdCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("mapreduce")
            .path("jobs").path(jobId).path("tasks").path(tid)
            .path("attempts").path(attid).path("counters")
            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("jobTaskAttemptCounters");
        verifyAMJobTaskAttemptCounters(info, att);
      }
    }
  }
}
 
Example 29
Source Project: XLearning   Source File: HistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetTaskReportResponse getTaskReport(GetTaskReportRequest request)
    throws IOException {
  TaskId taskId = request.getTaskId();
  Job job = verifyAndGetJob(taskId.getJobId(), true);
  GetTaskReportResponse response = recordFactory.newRecordInstance(GetTaskReportResponse.class);
  response.setTaskReport(job.getTask(taskId).getReport());
  return response;
}
 
Example 30
Source Project: big-c   Source File: TestHsWebServicesJobsQuery.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobsQueryFinishTimeBeginEnd() throws JSONException, Exception {
  WebResource r = resource();

  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  int size = jobsMap.size();
  // figure out the mid end time - we expect atleast 3 jobs
  ArrayList<Long> finishTime = new ArrayList<Long>(size);
  for (Map.Entry<JobId, Job> entry : jobsMap.entrySet()) {
    finishTime.add(entry.getValue().getReport().getFinishTime());
  }
  Collections.sort(finishTime);

  assertTrue("Error we must have atleast 3 jobs", size >= 3);
  long midFinishTime = finishTime.get(size - 2);

  ClientResponse response = r.path("ws").path("v1").path("history")
      .path("mapreduce").path("jobs")
      .queryParam("finishedTimeBegin", String.valueOf(40000))
      .queryParam("finishedTimeEnd", String.valueOf(midFinishTime))
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject jobs = json.getJSONObject("jobs");
  JSONArray arr = jobs.getJSONArray("job");
  assertEquals("incorrect number of elements", size - 1, arr.length());
}