Java Code Examples for org.apache.hadoop.mapreduce.v2.util.MRApps#toString()

The following examples show how to use org.apache.hadoop.mapreduce.v2.util.MRApps#toString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestAMWebServicesAttempts.java From hadoop with Apache License 2.0

6 votes

public void verifyAMTaskAttemptsXML(NodeList nodes, Task task) {
  assertEquals("incorrect number of elements", 1, nodes.getLength());

  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;
    for (int i = 0; i < nodes.getLength(); i++) {
      Element element = (Element) nodes.item(i);

      if (attid.matches(WebServicesTestUtils.getXmlString(element, "id"))) {
        found = true;
        verifyAMTaskAttemptXML(element, att, task.getType());
      }
    }
    assertTrue("task with id: " + attid + " not in web service output", found);
  }
}

Example 2

Source File: TestAMWebServicesJobs.java From big-c with Apache License 2.0

6 votes

@Test
public void testJobCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("counters")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    verifyAMJobCounters(info, jobsMap.get(id));
  }
}

Example 3

Source File: TestAMWebServicesTasks.java From big-c with Apache License 2.0

6 votes

@Test
public void testTasksQueryMap() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    String type = "m";
    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("tasks").queryParam("type", type)
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject tasks = json.getJSONObject("tasks");
    JSONArray arr = tasks.getJSONArray("task");
    assertEquals("incorrect number of elements", 1, arr.length());
    verifyAMTask(arr, jobsMap.get(id), type);
  }
}

Example 4

Source File: TestAMWebServicesJobs.java From hadoop with Apache License 2.0

6 votes

@Test
public void testJobIdSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId + "/").accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("job");
    verifyAMJob(info, jobsMap.get(id));
  }
}

Example 5

Source File: TestAMWebServicesTasks.java From hadoop with Apache License 2.0

6 votes

@Test
public void testTasksQueryMap() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    String type = "m";
    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("tasks").queryParam("type", type)
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject tasks = json.getJSONObject("tasks");
    JSONArray arr = tasks.getJSONArray("task");
    assertEquals("incorrect number of elements", 1, arr.length());
    verifyAMTask(arr, jobsMap.get(id), type);
  }
}

Example 6

Source File: TestAMWebServicesJobs.java From hadoop with Apache License 2.0

6 votes

@Test
public void testJobAttemptsDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1")
        .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobAttempts");
    verifyJobAttempts(info, jobsMap.get(id));
  }
}

Example 7

Source File: TestHsWebServicesAttempts.java From big-c with Apache License 2.0

6 votes

public void verifyHsTaskAttempts(JSONObject json, Task task)
    throws JSONException {
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject attempts = json.getJSONObject("taskAttempts");
  assertEquals("incorrect number of elements", 1, json.length());
  JSONArray arr = attempts.getJSONArray("taskAttempt");
  for (TaskAttempt att : task.getAttempts().values()) {
    TaskAttemptId id = att.getID();
    String attid = MRApps.toString(id);
    Boolean found = false;

    for (int i = 0; i < arr.length(); i++) {
      JSONObject info = arr.getJSONObject(i);
      if (attid.matches(info.getString("id"))) {
        found = true;
        verifyHsTaskAttempt(info, att, task.getType());
      }
    }
    assertTrue("task attempt with id: " + attid
        + " not in web service output", found);
  }
}

Example 8

Source File: TestAMWebServicesTasks.java From big-c with Apache License 2.0

6 votes

public void verifyTaskGeneric(Task task, String id, String state,
    String type, String successfulAttempt, long startTime, long finishTime,
    long elapsedTime, float progress, String status) {

  TaskId taskid = task.getID();
  String tid = MRApps.toString(taskid);
  TaskReport report = task.getReport();

  WebServicesTestUtils.checkStringMatch("id", tid, id);
  WebServicesTestUtils.checkStringMatch("type", task.getType().toString(),
      type);
  WebServicesTestUtils.checkStringMatch("state", report.getTaskState()
      .toString(), state);
  // not easily checked without duplicating logic, just make sure its here
  assertNotNull("successfulAttempt null", successfulAttempt);
  assertEquals("startTime wrong", report.getStartTime(), startTime);
  assertEquals("finishTime wrong", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f);
  assertEquals("status wrong", report.getStatus(), status);
}

Example 9

Source File: TestHsWebServicesJobs.java From big-c with Apache License 2.0

6 votes

@Test
public void testJobIdDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("job");
    VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id));
  }

}

Example 10

Source File: TestAMWebServicesJobs.java From big-c with Apache License 2.0

6 votes

@Test
public void testJobIdSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId + "/").accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("job");
    verifyAMJob(info, jobsMap.get(id));
  }
}

Example 11

Source File: TestAMWebServicesJobs.java From hadoop with Apache License 2.0

6 votes

@Test
public void testJobCountersDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).path("counters/").get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    verifyAMJobCounters(info, jobsMap.get(id));
  }
}

Example 12

Source File: TestAMWebServicesJobs.java From hadoop with Apache License 2.0

6 votes

@Test
public void testJobIdDefault() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("mapreduce")
        .path("jobs").path(jobId).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("job");
    verifyAMJob(info, jobsMap.get(id));
  }

}

Example 13

Source File: TestHsWebServicesJobs.java From hadoop with Apache License 2.0

6 votes

@Test
public void testJobCounters() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    ClientResponse response = r.path("ws").path("v1").path("history")
        .path("mapreduce").path("jobs").path(jobId).path("counters")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject info = json.getJSONObject("jobCounters");
    verifyHsJobCounters(info, appContext.getJob(id));
  }
}

Example 14

Source File: TestAMWebServicesTasks.java From big-c with Apache License 2.0

6 votes

@Test
public void testTaskId() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("mapreduce")
          .path("jobs").path(jobId).path("tasks").path(tid)
          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      assertEquals("incorrect number of elements", 1, json.length());
      JSONObject info = json.getJSONObject("task");
      verifyAMSingleTask(info, task);
    }
  }
}

Example 15

Source File: TestHsWebServicesTasks.java From big-c with Apache License 2.0

5 votes

@Test
public void testTaskIdXML() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    for (Task task : jobsMap.get(id).getTasks().values()) {

      String tid = MRApps.toString(task.getID());
      ClientResponse response = r.path("ws").path("v1").path("history")
          .path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid)
          .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);

      assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
      String xml = response.getEntity(String.class);
      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
      DocumentBuilder db = dbf.newDocumentBuilder();
      InputSource is = new InputSource();
      is.setCharacterStream(new StringReader(xml));
      Document dom = db.parse(is);
      NodeList nodes = dom.getElementsByTagName("task");
      for (int i = 0; i < nodes.getLength(); i++) {
        Element element = (Element) nodes.item(i);
        verifyHsSingleTaskXML(element, task);
      }
    }
  }
}

Example 16

Source File: TestHsWebServicesTasks.java From big-c with Apache License 2.0

5 votes

@Test
public void testTaskIdInvalid2() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    String tid = "task_0000_m_000000";
    try {
      r.path("ws").path("v1").path("history").path("mapreduce").path("jobs")
          .path(jobId).path("tasks").path(tid).get(JSONObject.class);
      fail("should have thrown exception on invalid uri");
    } catch (UniformInterfaceException ue) {
      ClientResponse response = ue.getResponse();
      assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject msg = response.getEntity(JSONObject.class);
      JSONObject exception = msg.getJSONObject("RemoteException");
      assertEquals("incorrect number of elements", 3, exception.length());
      String message = exception.getString("message");
      String type = exception.getString("exception");
      String classname = exception.getString("javaClassName");
      WebServicesTestUtils.checkStringMatch("exception message",
          "java.lang.Exception: TaskId string : "
              + "task_0000_m_000000 is not properly formed", message);
      WebServicesTestUtils.checkStringMatch("exception type",
          "NotFoundException", type);
      WebServicesTestUtils.checkStringMatch("exception classname",
          "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
    }
  }
}

Example 17

Source File: TestHsWebServicesAttempts.java From big-c with Apache License 2.0

5 votes

public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}

Example 18

Source File: TestHsWebServicesTasks.java From hadoop with Apache License 2.0

5 votes

@Test
public void testTaskIdBogus() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();
  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);
    String tid = "bogustaskid";
    try {
      r.path("ws").path("v1").path("history").path("mapreduce").path("jobs")
          .path(jobId).path("tasks").path(tid).get(JSONObject.class);
      fail("should have thrown exception on invalid uri");
    } catch (UniformInterfaceException ue) {
      ClientResponse response = ue.getResponse();
      assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject msg = response.getEntity(JSONObject.class);
      JSONObject exception = msg.getJSONObject("RemoteException");
      assertEquals("incorrect number of elements", 3, exception.length());
      String message = exception.getString("message");
      String type = exception.getString("exception");
      String classname = exception.getString("javaClassName");
      WebServicesTestUtils.checkStringMatch("exception message",
          "java.lang.Exception: TaskId string : "
              + "bogustaskid is not properly formed", message);
      WebServicesTestUtils.checkStringMatch("exception type",
          "NotFoundException", type);
      WebServicesTestUtils.checkStringMatch("exception classname",
          "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
    }
  }
}

Example 19

Source File: TestHsWebServicesAttempts.java From hadoop with Apache License 2.0

5 votes

@Test
public void testTaskAttemptIdSlash() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("history")
            .path("mapreduce").path("jobs").path(jobId).path("tasks")
            .path(tid).path("attempts").path(attid + "/")
            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("taskAttempt");
        verifyHsTaskAttempt(info, att, task.getType());
      }
    }
  }
}

Example 20

Source File: TestAMWebServicesAttempts.java From hadoop with Apache License 2.0

5 votes

@Test
public void testTaskAttemptId() throws JSONException, Exception {
  WebResource r = resource();
  Map<JobId, Job> jobsMap = appContext.getAllJobs();

  for (JobId id : jobsMap.keySet()) {
    String jobId = MRApps.toString(id);

    for (Task task : jobsMap.get(id).getTasks().values()) {
      String tid = MRApps.toString(task.getID());

      for (TaskAttempt att : task.getAttempts().values()) {
        TaskAttemptId attemptid = att.getID();
        String attid = MRApps.toString(attemptid);

        ClientResponse response = r.path("ws").path("v1").path("mapreduce")
            .path("jobs").path(jobId).path("tasks").path(tid)
            .path("attempts").path(attid).accept(MediaType.APPLICATION_JSON)
            .get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("taskAttempt");
        verifyAMTaskAttempt(info, att, task.getType());
      }
    }
  }
}