org.apache.hadoop.mapreduce.server.jobtracker.JTConfig Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.server.jobtracker.JTConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestJobClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetClusterStatusWithLocalJobRunner() throws Exception {
  Configuration conf = new Configuration();
  conf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  JobClient client = new JobClient(conf);
  ClusterStatus clusterStatus = client.getClusterStatus(true);
  Collection<String> activeTrackerNames = clusterStatus
      .getActiveTrackerNames();
  Assert.assertEquals(0, activeTrackerNames.size());
  int blacklistedTrackers = clusterStatus.getBlacklistedTrackers();
  Assert.assertEquals(0, blacklistedTrackers);
  Collection<BlackListInfo> blackListedTrackersInfo = clusterStatus
      .getBlackListedTrackersInfo();
  Assert.assertEquals(0, blackListedTrackersInfo.size());
}
 
Example #2
Source File: TestMiniMRWithDFSWithDistinctUsers.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();

  fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
      public FileSystem run() throws IOException {
        return dfs.getFileSystem();
      }
    });
  // Home directories for users
  mkdir(fs, "/user", "nobody", "nogroup", (short)01777);
  mkdir(fs, "/user/alice", "alice", "nogroup", (short)0755);
  mkdir(fs, "/user/bob", "bob", "nogroup", (short)0755);

  // staging directory root with sticky bit
  UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
  mkdir(fs, "/staging", MR_UGI.getShortUserName(), "nogroup", (short)01777);

  JobConf mrConf = new JobConf();
  mrConf.set(JTConfig.JT_STAGING_AREA_ROOT, "/staging");

  mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
                         1, null, null, MR_UGI, mrConf);
}
 
Example #3
Source File: TestJobSysDirWithDFS.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void testWithDFS() throws IOException {
  MiniDFSCluster dfs = null;
  MiniMRCluster mr = null;
  FileSystem fileSys = null;
  try {
    final int taskTrackers = 4;

    JobConf conf = new JobConf();
    conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/custom/mapred/system");
    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fileSys = dfs.getFileSystem();
    mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, null, null, conf);

    runWordCount(mr, mr.createJobConf(), conf.get("mapred.system.dir"));
  } finally {
    if (dfs != null) { dfs.shutdown(); }
    if (mr != null) { mr.shutdown();
    }
  }
}
 
Example #4
Source File: TestStreamingStatus.java    From hadoop with Apache License 2.0 6 votes vote down vote up
void runStreamJob(TaskType type, boolean isEmptyInput) throws IOException {
  boolean mayExit = false;
  StreamJob job = new StreamJob(genArgs(
      mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS), map, reduce), mayExit);
  int returnValue = job.go();
  assertEquals(0, returnValue);

  // If input to reducer is empty, dummy reporter(which ignores all
  // reporting lines) is set for MRErrorThread in waitOutputThreads(). So
  // expectedCounterValue is 0 for empty-input-to-reducer case.
  // Output of reducer is also empty for empty-input-to-reducer case.
  int expectedCounterValue = 0;
  if (type == TaskType.MAP || !isEmptyInput) {
    validateTaskStatus(job, type);
    // output is from "print STDOUT" statements in perl script
    validateJobOutput(job.getConf());
    expectedCounterValue = 2;
  }
  validateUserCounter(job, expectedCounterValue);
  validateTaskStderr(job, type);

  deleteOutDir(fs);
}
 
Example #5
Source File: TestStreamingStatus.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected String[] genArgs(String jobtracker, String mapper, String reducer)
{
  return new String[] {
    "-input", INPUT_FILE,
    "-output", OUTPUT_DIR,
    "-mapper", mapper,
    "-reducer", reducer,
    "-jobconf", MRJobConfig.NUM_MAPS + "=1",
    "-jobconf", MRJobConfig.NUM_REDUCES + "=1",
    "-jobconf", MRJobConfig.PRESERVE_FAILED_TASK_FILES + "=true",
    "-jobconf", "stream.tmpdir=" + new Path(TEST_ROOT_DIR).toUri().getPath(),
    "-jobconf", JTConfig.JT_IPC_ADDRESS + "="+jobtracker,
    "-jobconf", "fs.default.name=file:///",
    "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
    "-jobconf", "mapreduce.framework.name=yarn"
  };
}
 
Example #6
Source File: TestMRFramework.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFramework() {
  JobConf jobConf = new JobConf();
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090");
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  assertTrue("Expected 'isLocal' to be true", 
      StreamUtil.isLocalJobTracker(jobConf));
}
 
Example #7
Source File: GridmixJob.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Sets the high ram job properties in the simulated job's configuration.
 */
@SuppressWarnings("deprecation")
static void configureHighRamProperties(Configuration sourceConf, 
                                       Configuration destConf) {
  // set the memory per map task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.MAPMEMORY_MB, MRJobConfig.MAP_MEMORY_MB, 
                       MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.MAP_MEMORY_MB, 
                           JTConfig.JT_MAX_MAPMEMORY_MB);
  
  // set the memory per reduce task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.REDUCEMEMORY_MB, MRJobConfig.REDUCE_MEMORY_MB,
                       MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.REDUCE_MEMORY_MB, 
                           JTConfig.JT_MAX_REDUCEMEMORY_MB);
}
 
Example #8
Source File: TestJobClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetClusterStatusWithLocalJobRunner() throws Exception {
  Configuration conf = new Configuration();
  conf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  JobClient client = new JobClient(conf);
  ClusterStatus clusterStatus = client.getClusterStatus(true);
  Collection<String> activeTrackerNames = clusterStatus
      .getActiveTrackerNames();
  Assert.assertEquals(0, activeTrackerNames.size());
  int blacklistedTrackers = clusterStatus.getBlacklistedTrackers();
  Assert.assertEquals(0, blacklistedTrackers);
  Collection<BlackListInfo> blackListedTrackersInfo = clusterStatus
      .getBlackListedTrackersInfo();
  Assert.assertEquals(0, blackListedTrackersInfo.size());
}
 
Example #9
Source File: GridmixJob.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Sets the high ram job properties in the simulated job's configuration.
 */
@SuppressWarnings("deprecation")
static void configureHighRamProperties(Configuration sourceConf, 
                                       Configuration destConf) {
  // set the memory per map task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.MAPMEMORY_MB, MRJobConfig.MAP_MEMORY_MB, 
                       MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.MAP_MEMORY_MB, 
                           JTConfig.JT_MAX_MAPMEMORY_MB);
  
  // set the memory per reduce task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.REDUCEMEMORY_MB, MRJobConfig.REDUCE_MEMORY_MB,
                       MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.REDUCE_MEMORY_MB, 
                           JTConfig.JT_MAX_REDUCEMEMORY_MB);
}
 
Example #10
Source File: TestMRFramework.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFramework() {
  JobConf jobConf = new JobConf();
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090");
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  assertTrue("Expected 'isLocal' to be true", 
      StreamUtil.isLocalJobTracker(jobConf));
}
 
Example #11
Source File: TestJobSysDirWithDFS.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testWithDFS() throws IOException {
  MiniDFSCluster dfs = null;
  MiniMRCluster mr = null;
  FileSystem fileSys = null;
  try {
    final int taskTrackers = 4;

    JobConf conf = new JobConf();
    conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/custom/mapred/system");
    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fileSys = dfs.getFileSystem();
    mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, null, null, conf);

    runWordCount(mr, mr.createJobConf(), conf.get("mapred.system.dir"));
  } finally {
    if (dfs != null) { dfs.shutdown(); }
    if (mr != null) { mr.shutdown();
    }
  }
}
 
Example #12
Source File: TestStreamingStatus.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected String[] genArgs(String jobtracker, String mapper, String reducer)
{
  return new String[] {
    "-input", INPUT_FILE,
    "-output", OUTPUT_DIR,
    "-mapper", mapper,
    "-reducer", reducer,
    "-jobconf", MRJobConfig.NUM_MAPS + "=1",
    "-jobconf", MRJobConfig.NUM_REDUCES + "=1",
    "-jobconf", MRJobConfig.PRESERVE_FAILED_TASK_FILES + "=true",
    "-jobconf", "stream.tmpdir=" + new Path(TEST_ROOT_DIR).toUri().getPath(),
    "-jobconf", JTConfig.JT_IPC_ADDRESS + "="+jobtracker,
    "-jobconf", "fs.default.name=file:///",
    "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
    "-jobconf", "mapreduce.framework.name=yarn"
  };
}
 
Example #13
Source File: TestStreamingStatus.java    From big-c with Apache License 2.0 6 votes vote down vote up
void runStreamJob(TaskType type, boolean isEmptyInput) throws IOException {
  boolean mayExit = false;
  StreamJob job = new StreamJob(genArgs(
      mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS), map, reduce), mayExit);
  int returnValue = job.go();
  assertEquals(0, returnValue);

  // If input to reducer is empty, dummy reporter(which ignores all
  // reporting lines) is set for MRErrorThread in waitOutputThreads(). So
  // expectedCounterValue is 0 for empty-input-to-reducer case.
  // Output of reducer is also empty for empty-input-to-reducer case.
  int expectedCounterValue = 0;
  if (type == TaskType.MAP || !isEmptyInput) {
    validateTaskStatus(job, type);
    // output is from "print STDOUT" statements in perl script
    validateJobOutput(job.getConf());
    expectedCounterValue = 2;
  }
  validateUserCounter(job, expectedCounterValue);
  validateTaskStderr(job, type);

  deleteOutDir(fs);
}
 
Example #14
Source File: TestMiniMRWithDFSWithDistinctUsers.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();

  fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
      public FileSystem run() throws IOException {
        return dfs.getFileSystem();
      }
    });
  // Home directories for users
  mkdir(fs, "/user", "nobody", "nogroup", (short)01777);
  mkdir(fs, "/user/alice", "alice", "nogroup", (short)0755);
  mkdir(fs, "/user/bob", "bob", "nogroup", (short)0755);

  // staging directory root with sticky bit
  UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); 
  mkdir(fs, "/staging", MR_UGI.getShortUserName(), "nogroup", (short)01777);

  JobConf mrConf = new JobConf();
  mrConf.set(JTConfig.JT_STAGING_AREA_ROOT, "/staging");

  mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
                         1, null, null, MR_UGI, mrConf);
}
 
Example #15
Source File: GridmixTestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static void initCluster(Class<?> caller) throws IOException {
    Configuration conf = new Configuration();
//    conf.set("mapred.queue.names", "default,q1,q2");
  conf.set("mapred.queue.names", "default");
    conf.set("yarn.scheduler.capacity.root.queues", "default");
    conf.set("yarn.scheduler.capacity.root.default.capacity", "100.0");
    
    
    conf.setBoolean(GRIDMIX_USE_QUEUE_IN_TRACE, false);
    conf.set(GRIDMIX_DEFAULT_QUEUE, "default");
    

    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
        .build();// MiniDFSCluster(conf, 3, true, null);
    dfs = dfsCluster.getFileSystem();
    conf.set(JTConfig.JT_RETIREJOBS, "false");
    mrvl = MiniMRClientClusterFactory.create(caller, 2, conf);
    
    conf = mrvl.getConfig();
    String[] files = conf.getStrings(MRJobConfig.CACHE_FILES);
    if (files != null) {
      String[] timestamps = new String[files.length];
      for (int i = 0; i < files.length; i++) {
        timestamps[i] = Long.toString(System.currentTimeMillis());
      }
      conf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps);
    }
    
  }
 
Example #16
Source File: TestGridmixSummary.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test {@link ClusterSummarizer}.
 */
@Test  (timeout=20000)
public void testClusterSummarizer() throws IOException {
  ClusterSummarizer cs = new ClusterSummarizer();
  Configuration conf = new Configuration();
  
  String jt = "test-jt:1234";
  String nn = "test-nn:5678";
  conf.set(JTConfig.JT_IPC_ADDRESS, jt);
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, nn);
  cs.start(conf);
  
  assertEquals("JT name mismatch", jt, cs.getJobTrackerInfo());
  assertEquals("NN name mismatch", nn, cs.getNamenodeInfo());
  
  ClusterStats cStats = ClusterStats.getClusterStats();
  conf.set(JTConfig.JT_IPC_ADDRESS, "local");
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "local");
  JobClient jc = new JobClient(conf);
  cStats.setClusterMetric(jc.getClusterStatus());
  
  cs.update(cStats);
  
  // test
  assertEquals("Cluster summary test failed!", 1, cs.getMaxMapTasks());
  assertEquals("Cluster summary test failed!", 1, cs.getMaxReduceTasks());
  assertEquals("Cluster summary test failed!", 1, cs.getNumActiveTrackers());
  assertEquals("Cluster summary test failed!", 0, 
               cs.getNumBlacklistedTrackers());
}
 
Example #17
Source File: TestMiniMRClasspath.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void configureWordCount(FileSystem fs, JobConf conf, String input,
    int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
  fs.delete(outDir, true);
  if (!fs.mkdirs(inDir)) {
    throw new IOException("Mkdirs failed to create " + inDir.toString());
  }
  DataOutputStream file = fs.create(new Path(inDir, "part-0"));
  file.writeBytes(input);
  file.close();
  FileSystem.setDefaultUri(conf, fs.getUri());
  conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
  conf.setJobName("wordcount");
  conf.setInputFormat(TextInputFormat.class);
  
  // the keys are words (strings)
  conf.setOutputKeyClass(Text.class);
  // the values are counts (ints)
  conf.setOutputValueClass(IntWritable.class);
  
  conf.set("mapred.mapper.class", "testjar.ClassWordCount$MapClass");        
  conf.set("mapred.combine.class", "testjar.ClassWordCount$Reduce");
  conf.set("mapred.reducer.class", "testjar.ClassWordCount$Reduce");
  FileInputFormat.setInputPaths(conf, inDir);
  FileOutputFormat.setOutputPath(conf, outDir);
  conf.setNumMapTasks(numMaps);
  conf.setNumReduceTasks(numReduces);
  //set the tests jar file
  conf.setJarByClass(TestMiniMRClasspath.class);
}
 
Example #18
Source File: TestMRWithDistributedCache.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Tests using the local job runner. */
public void testLocalJobRunner() throws Exception {
  symlinkFile.delete(); // ensure symlink is not present (e.g. if test is
                        // killed part way through)
  
  Configuration c = new Configuration();
  c.set(JTConfig.JT_IPC_ADDRESS, "local");
  c.set("fs.defaultFS", "file:///");
  testWithConf(c);
  
  assertFalse("Symlink not removed by local job runner",
          // Symlink target will have gone so can't use File.exists()
          Arrays.asList(new File(".").list()).contains(symlinkFile.getName()));
}
 
Example #19
Source File: LocalJobRunner.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * @see org.apache.hadoop.mapreduce.protocol.ClientProtocol#getStagingAreaDir()
 */
public String getStagingAreaDir() throws IOException {
  Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT, 
      "/tmp/hadoop/mapred/staging"));
  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  String user;
  randid = rand.nextInt(Integer.MAX_VALUE);
  if (ugi != null) {
    user = ugi.getShortUserName() + randid;
  } else {
    user = "dummy" + randid;
  }
  return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString();
}
 
Example #20
Source File: TestClientProtocolProviderImpls.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testClusterWithJTClientProvider() throws Exception {
  Configuration conf = new Configuration();
  try {
    conf.set(MRConfig.FRAMEWORK_NAME, "classic");
    conf.set(JTConfig.JT_IPC_ADDRESS, "local");
    new Cluster(conf);
    fail("Cluster with classic Framework name should not use "
        + "local JT address");
  } catch (IOException e) {
    assertTrue(e.getMessage().contains(
        "Cannot initialize Cluster. Please check"));
  }
}
 
Example #21
Source File: TestStreamingStatus.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Start the cluster and create input file before running the actual test.
 *
 * @throws IOException
 */
@Before
public void setUp() throws IOException {
  conf = new JobConf();
  conf.setBoolean(JTConfig.JT_RETIREJOBS, false);
  conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);

  mr = new MiniMRCluster(1, "file:///", 3, null , null, conf);

  Path inFile = new Path(INPUT_FILE);
  fs = inFile.getFileSystem(mr.createJobConf());
  clean(fs);

  buildExpectedJobOutput();
}
 
Example #22
Source File: ReliabilityTest.java    From big-c with Apache License 2.0 5 votes vote down vote up
public int run(String[] args) throws Exception {
  Configuration conf = getConf();
  if ("local".equals(conf.get(JTConfig.JT_IPC_ADDRESS, "local"))) {
    displayUsage();
  }
  String[] otherArgs = 
    new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length == 2) {
    if (otherArgs[0].equals("-scratchdir")) {
      dir = otherArgs[1];
    } else {
      displayUsage();
    }
  }
  else if (otherArgs.length == 0) {
    dir = System.getProperty("user.dir");
  } else {
    displayUsage();
  }
  
  //to protect against the case of jobs failing even when multiple attempts
  //fail, set some high values for the max attempts
  conf.setInt(JobContext.MAP_MAX_ATTEMPTS, 10);
  conf.setInt(JobContext.REDUCE_MAX_ATTEMPTS, 10);
  runSleepJobTest(new JobClient(new JobConf(conf)), conf);
  runSortJobTests(new JobClient(new JobConf(conf)), conf);
  return 0;
}
 
Example #23
Source File: TestStreamingStatus.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Start the cluster and create input file before running the actual test.
 *
 * @throws IOException
 */
@Before
public void setUp() throws IOException {
  conf = new JobConf();
  conf.setBoolean(JTConfig.JT_RETIREJOBS, false);
  conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);

  mr = new MiniMRCluster(1, "file:///", 3, null , null, conf);

  Path inFile = new Path(INPUT_FILE);
  fs = inFile.getFileSystem(mr.createJobConf());
  clean(fs);

  buildExpectedJobOutput();
}
 
Example #24
Source File: ReliabilityTest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public int run(String[] args) throws Exception {
  Configuration conf = getConf();
  if ("local".equals(conf.get(JTConfig.JT_IPC_ADDRESS, "local"))) {
    displayUsage();
  }
  String[] otherArgs = 
    new GenericOptionsParser(conf, args).getRemainingArgs();
  if (otherArgs.length == 2) {
    if (otherArgs[0].equals("-scratchdir")) {
      dir = otherArgs[1];
    } else {
      displayUsage();
    }
  }
  else if (otherArgs.length == 0) {
    dir = System.getProperty("user.dir");
  } else {
    displayUsage();
  }
  
  //to protect against the case of jobs failing even when multiple attempts
  //fail, set some high values for the max attempts
  conf.setInt(JobContext.MAP_MAX_ATTEMPTS, 10);
  conf.setInt(JobContext.REDUCE_MAX_ATTEMPTS, 10);
  runSleepJobTest(new JobClient(new JobConf(conf)), conf);
  runSortJobTests(new JobClient(new JobConf(conf)), conf);
  return 0;
}
 
Example #25
Source File: TestClientProtocolProviderImpls.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testClusterWithJTClientProvider() throws Exception {
  Configuration conf = new Configuration();
  try {
    conf.set(MRConfig.FRAMEWORK_NAME, "classic");
    conf.set(JTConfig.JT_IPC_ADDRESS, "local");
    new Cluster(conf);
    fail("Cluster with classic Framework name should not use "
        + "local JT address");
  } catch (IOException e) {
    assertTrue(e.getMessage().contains(
        "Cannot initialize Cluster. Please check"));
  }
}
 
Example #26
Source File: TestGridmixSummary.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test {@link ClusterSummarizer}.
 */
@Test  (timeout=20000)
public void testClusterSummarizer() throws IOException {
  ClusterSummarizer cs = new ClusterSummarizer();
  Configuration conf = new Configuration();
  
  String jt = "test-jt:1234";
  String nn = "test-nn:5678";
  conf.set(JTConfig.JT_IPC_ADDRESS, jt);
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, nn);
  cs.start(conf);
  
  assertEquals("JT name mismatch", jt, cs.getJobTrackerInfo());
  assertEquals("NN name mismatch", nn, cs.getNamenodeInfo());
  
  ClusterStats cStats = ClusterStats.getClusterStats();
  conf.set(JTConfig.JT_IPC_ADDRESS, "local");
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "local");
  JobClient jc = new JobClient(conf);
  cStats.setClusterMetric(jc.getClusterStatus());
  
  cs.update(cStats);
  
  // test
  assertEquals("Cluster summary test failed!", 1, cs.getMaxMapTasks());
  assertEquals("Cluster summary test failed!", 1, cs.getMaxReduceTasks());
  assertEquals("Cluster summary test failed!", 1, cs.getNumActiveTrackers());
  assertEquals("Cluster summary test failed!", 0, 
               cs.getNumBlacklistedTrackers());
}
 
Example #27
Source File: GridmixTestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void initCluster(Class<?> caller) throws IOException {
    Configuration conf = new Configuration();
//    conf.set("mapred.queue.names", "default,q1,q2");
  conf.set("mapred.queue.names", "default");
    conf.set("yarn.scheduler.capacity.root.queues", "default");
    conf.set("yarn.scheduler.capacity.root.default.capacity", "100.0");
    
    
    conf.setBoolean(GRIDMIX_USE_QUEUE_IN_TRACE, false);
    conf.set(GRIDMIX_DEFAULT_QUEUE, "default");
    

    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
        .build();// MiniDFSCluster(conf, 3, true, null);
    dfs = dfsCluster.getFileSystem();
    conf.set(JTConfig.JT_RETIREJOBS, "false");
    mrvl = MiniMRClientClusterFactory.create(caller, 2, conf);
    
    conf = mrvl.getConfig();
    String[] files = conf.getStrings(MRJobConfig.CACHE_FILES);
    if (files != null) {
      String[] timestamps = new String[files.length];
      for (int i = 0; i < files.length; i++) {
        timestamps[i] = Long.toString(System.currentTimeMillis());
      }
      conf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps);
    }
    
  }
 
Example #28
Source File: TestMiniMRClasspath.java    From hadoop with Apache License 2.0 5 votes vote down vote up
static void configureWordCount(FileSystem fs, JobConf conf, String input,
    int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
  fs.delete(outDir, true);
  if (!fs.mkdirs(inDir)) {
    throw new IOException("Mkdirs failed to create " + inDir.toString());
  }
  DataOutputStream file = fs.create(new Path(inDir, "part-0"));
  file.writeBytes(input);
  file.close();
  FileSystem.setDefaultUri(conf, fs.getUri());
  conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
  conf.setJobName("wordcount");
  conf.setInputFormat(TextInputFormat.class);
  
  // the keys are words (strings)
  conf.setOutputKeyClass(Text.class);
  // the values are counts (ints)
  conf.setOutputValueClass(IntWritable.class);
  
  conf.set("mapred.mapper.class", "testjar.ClassWordCount$MapClass");        
  conf.set("mapred.combine.class", "testjar.ClassWordCount$Reduce");
  conf.set("mapred.reducer.class", "testjar.ClassWordCount$Reduce");
  FileInputFormat.setInputPaths(conf, inDir);
  FileOutputFormat.setOutputPath(conf, outDir);
  conf.setNumMapTasks(numMaps);
  conf.setNumReduceTasks(numReduces);
  //set the tests jar file
  conf.setJarByClass(TestMiniMRClasspath.class);
}
 
Example #29
Source File: TestMRWithDistributedCache.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Tests using the local job runner. */
public void testLocalJobRunner() throws Exception {
  symlinkFile.delete(); // ensure symlink is not present (e.g. if test is
                        // killed part way through)
  
  Configuration c = new Configuration();
  c.set(JTConfig.JT_IPC_ADDRESS, "local");
  c.set("fs.defaultFS", "file:///");
  testWithConf(c);
  
  assertFalse("Symlink not removed by local job runner",
          // Symlink target will have gone so can't use File.exists()
          Arrays.asList(new File(".").list()).contains(symlinkFile.getName()));
}
 
Example #30
Source File: LocalJobRunner.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * @see org.apache.hadoop.mapreduce.protocol.ClientProtocol#getStagingAreaDir()
 */
public String getStagingAreaDir() throws IOException {
  Path stagingRootDir = new Path(conf.get(JTConfig.JT_STAGING_AREA_ROOT, 
      "/tmp/hadoop/mapred/staging"));
  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  String user;
  randid = rand.nextInt(Integer.MAX_VALUE);
  if (ugi != null) {
    user = ugi.getShortUserName() + randid;
  } else {
    user = "dummy" + randid;
  }
  return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString();
}