Java Code Examples for org.apache.hadoop.mapred.MRCaching.TestResult

The following examples show how to use org.apache.hadoop.mapred.MRCaching.TestResult. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestMiniMRDFSCaching.java    License: Apache License 2.0 5 votes vote down vote up
public void testWithDFS() throws IOException {
  MiniMRCluster mr = null;
  MiniDFSCluster dfs = null;
  FileSystem fileSys = null;
  try {
    JobConf conf = new JobConf();
    dfs = new MiniDFSCluster.Builder(conf).build();
    fileSys = dfs.getFileSystem();
    mr = new MiniMRCluster(2, fileSys.getUri().toString(), 4);
    MRCaching.setupCache("/cachedir", fileSys);
    // run the wordcount example with caching
    TestResult ret = MRCaching.launchMRCache("/testing/wc/input",
                                          "/testing/wc/output",
                                          "/cachedir",
                                          mr.createJobConf(),
                                          "The quick brown fox\nhas many silly\n"
                                          + "red fox sox\n");
    assertTrue("Archives not matching", ret.isOutputOk);
    // launch MR cache with symlinks
    ret = MRCaching.launchMRCache("/testing/wc/input",
                                  "/testing/wc/output",
                                  "/cachedir",
                                  mr.createJobConf(),
                                  "The quick brown fox\nhas many silly\n"
                                  + "red fox sox\n");
    assertTrue("Archives not matching", ret.isOutputOk);
  } finally {
    if (fileSys != null) {
      fileSys.close();
    }
    if (dfs != null) {
      dfs.shutdown();
    }
    if (mr != null) {
      mr.shutdown();
    }
  }
}
 
Example 2
Source Project: big-c   Source File: TestMiniMRDFSCaching.java    License: Apache License 2.0 5 votes vote down vote up
public void testWithDFS() throws IOException {
  MiniMRCluster mr = null;
  MiniDFSCluster dfs = null;
  FileSystem fileSys = null;
  try {
    JobConf conf = new JobConf();
    dfs = new MiniDFSCluster.Builder(conf).build();
    fileSys = dfs.getFileSystem();
    mr = new MiniMRCluster(2, fileSys.getUri().toString(), 4);
    MRCaching.setupCache("/cachedir", fileSys);
    // run the wordcount example with caching
    TestResult ret = MRCaching.launchMRCache("/testing/wc/input",
                                          "/testing/wc/output",
                                          "/cachedir",
                                          mr.createJobConf(),
                                          "The quick brown fox\nhas many silly\n"
                                          + "red fox sox\n");
    assertTrue("Archives not matching", ret.isOutputOk);
    // launch MR cache with symlinks
    ret = MRCaching.launchMRCache("/testing/wc/input",
                                  "/testing/wc/output",
                                  "/cachedir",
                                  mr.createJobConf(),
                                  "The quick brown fox\nhas many silly\n"
                                  + "red fox sox\n");
    assertTrue("Archives not matching", ret.isOutputOk);
  } finally {
    if (fileSys != null) {
      fileSys.close();
    }
    if (dfs != null) {
      dfs.shutdown();
    }
    if (mr != null) {
      mr.shutdown();
    }
  }
}
 
Example 3
Source Project: RDFS   Source File: TestMiniMRLocalFS.java    License: Apache License 2.0 4 votes vote down vote up
public void testWithLocal()
    throws IOException, InterruptedException, ClassNotFoundException {
  MiniMRCluster mr = null;
  try {
    mr = new MiniMRCluster(2, "file:///", 3);
    TestMiniMRWithDFS.runPI(mr, mr.createJobConf());

    // run the wordcount example with caching
    JobConf job = mr.createJobConf();
    TestResult ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input",
                                          TEST_ROOT_DIR + "/wc/output", 
                                          TEST_ROOT_DIR + "/cachedir",
                                          job,
                                          "The quick brown fox\n" 
                                          + "has many silly\n"
                                          + "red fox sox\n");
    // assert the number of lines read during caching
    assertTrue("Failed test archives not matching", ret.isOutputOk);
    // test the task report fetchers
    JobClient client = new JobClient(job);
    JobID jobid = ret.job.getID();
    TaskReport[] reports;
    reports = client.getSetupTaskReports(jobid);
    assertEquals("number of setups", 2, reports.length);
    reports = client.getMapTaskReports(jobid);
    assertEquals("number of maps", 1, reports.length);
    reports = client.getReduceTaskReports(jobid);
    assertEquals("number of reduces", 1, reports.length);
    reports = client.getCleanupTaskReports(jobid);
    assertEquals("number of cleanups", 2, reports.length);
    Counters counters = ret.job.getCounters();
    assertEquals("number of map inputs", 3, 
                 counters.getCounter(Task.Counter.MAP_INPUT_RECORDS));
    assertEquals("number of reduce outputs", 9, 
                 counters.getCounter(Task.Counter.REDUCE_OUTPUT_RECORDS));
    runCustomFormats(mr);
    runSecondarySort(mr.createJobConf());
  } finally {
    if (mr != null) { mr.shutdown(); }
  }
}
 
Example 4
Source Project: hadoop-gpu   Source File: TestMiniMRLocalFS.java    License: Apache License 2.0 4 votes vote down vote up
public void testWithLocal()
    throws IOException, InterruptedException, ClassNotFoundException {
  MiniMRCluster mr = null;
  try {
    mr = new MiniMRCluster(2, "file:///", 3);
    TestMiniMRWithDFS.runPI(mr, mr.createJobConf());

    // run the wordcount example with caching
    JobConf job = mr.createJobConf();
    TestResult ret = MRCaching.launchMRCache(TEST_ROOT_DIR + "/wc/input",
                                          TEST_ROOT_DIR + "/wc/output", 
                                          TEST_ROOT_DIR + "/cachedir",
                                          job,
                                          "The quick brown fox\n" 
                                          + "has many silly\n"
                                          + "red fox sox\n");
    // assert the number of lines read during caching
    assertTrue("Failed test archives not matching", ret.isOutputOk);
    // test the task report fetchers
    JobClient client = new JobClient(job);
    JobID jobid = ret.job.getID();
    TaskReport[] reports;
    reports = client.getSetupTaskReports(jobid);
    assertEquals("number of setups", 2, reports.length);
    reports = client.getMapTaskReports(jobid);
    assertEquals("number of maps", 1, reports.length);
    reports = client.getReduceTaskReports(jobid);
    assertEquals("number of reduces", 1, reports.length);
    reports = client.getCleanupTaskReports(jobid);
    assertEquals("number of cleanups", 2, reports.length);
    Counters counters = ret.job.getCounters();
    assertEquals("number of map inputs", 3, 
                 counters.getCounter(Task.Counter.MAP_INPUT_RECORDS));
    assertEquals("number of reduce outputs", 9, 
                 counters.getCounter(Task.Counter.REDUCE_OUTPUT_RECORDS));
    runCustomFormats(mr);
    runSecondarySort(mr.createJobConf());
  } finally {
    if (mr != null) { mr.shutdown(); }
  }
}