org.apache.hadoop.mapreduce.MRConfig Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.MRConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
   Configuration conf = new Configuration();
   conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
   conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
   Map<String, String> env = new HashMap<String, String>();
   try {
     MRApps.setClasspath(env, conf);
   } catch (Exception e) {
     fail("Got exception while setting classpath");
   }
   String env_str = env.get("CLASSPATH");
   String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
     Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
       "job.jar/classes/", "job.jar/lib/*",
       ApplicationConstants.Environment.PWD.$$() + "/*"));
   assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
     env_str.startsWith(expectedClasspath));
 }
 
Example #2
Source File: TestJobAclsManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGroups() {
  Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
  Configuration conf = new Configuration();
  String jobOwner = "testuser";
  conf.set(JobACL.VIEW_JOB.getAclName(), jobOwner);
  conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
  String user = "testuser2";
  String adminGroup = "adminGroup";
  conf.set(MRConfig.MR_ADMINS, " " + adminGroup);

  JobACLsManager aclsManager = new JobACLsManager(conf);
  tmpJobACLs = aclsManager.constructJobACLs(conf);
  final Map<JobACL, AccessControlList> jobACLs = tmpJobACLs;

  UserGroupInformation callerUGI = UserGroupInformation.createUserForTesting(
   user, new String[] {adminGroup});
  // acls off so anyone should have access
  boolean val = aclsManager.checkAccess(callerUGI, JobACL.VIEW_JOB, jobOwner,
      jobACLs.get(JobACL.VIEW_JOB));
  assertTrue("user in admin group should have access", val);
}
 
Example #3
Source File: GridmixJob.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Sets the high ram job properties in the simulated job's configuration.
 */
@SuppressWarnings("deprecation")
static void configureHighRamProperties(Configuration sourceConf, 
                                       Configuration destConf) {
  // set the memory per map task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.MAPMEMORY_MB, MRJobConfig.MAP_MEMORY_MB, 
                       MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.MAP_MEMORY_MB, 
                           JTConfig.JT_MAX_MAPMEMORY_MB);
  
  // set the memory per reduce task
  scaleConfigParameter(sourceConf, destConf, 
                       MRConfig.REDUCEMEMORY_MB, MRJobConfig.REDUCE_MEMORY_MB,
                       MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  // validate and fail early
  validateTaskMemoryLimits(destConf, MRJobConfig.REDUCE_MEMORY_MB, 
                           JTConfig.JT_MAX_REDUCEMEMORY_MB);
}
 
Example #4
Source File: TestJobSplitWriter.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxBlockLocationsOldSplits() throws Exception {
  TEST_DIR.mkdirs();
  try {
    Configuration conf = new Configuration();
    conf.setInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY, 4);
    Path submitDir = new Path(TEST_DIR.getAbsolutePath());
    FileSystem fs = FileSystem.getLocal(conf);
    org.apache.hadoop.mapred.FileSplit split =
        new org.apache.hadoop.mapred.FileSplit(new Path("/some/path"), 0, 1,
            new String[] { "loc1", "loc2", "loc3", "loc4", "loc5" });
    JobSplitWriter.createSplitFiles(submitDir, conf, fs,
        new org.apache.hadoop.mapred.InputSplit[] { split });
    JobSplit.TaskSplitMetaInfo[] infos =
        SplitMetaInfoReader.readSplitMetaInfo(new JobID(), fs, conf,
            submitDir);
    assertEquals("unexpected number of splits", 1, infos.length);
    assertEquals("unexpected number of split locations",
        4, infos[0].getLocations().length);
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 
Example #5
Source File: TestMRFramework.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFramework() {
  JobConf jobConf = new JobConf();
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090");
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  assertTrue("Expected 'isLocal' to be true", 
      StreamUtil.isLocalJobTracker(jobConf));
}
 
Example #6
Source File: TestJobSplitWriter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxBlockLocationsOldSplits() throws Exception {
  TEST_DIR.mkdirs();
  try {
    Configuration conf = new Configuration();
    conf.setInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY, 4);
    Path submitDir = new Path(TEST_DIR.getAbsolutePath());
    FileSystem fs = FileSystem.getLocal(conf);
    org.apache.hadoop.mapred.FileSplit split =
        new org.apache.hadoop.mapred.FileSplit(new Path("/some/path"), 0, 1,
            new String[] { "loc1", "loc2", "loc3", "loc4", "loc5" });
    JobSplitWriter.createSplitFiles(submitDir, conf, fs,
        new org.apache.hadoop.mapred.InputSplit[] { split });
    JobSplit.TaskSplitMetaInfo[] infos =
        SplitMetaInfoReader.readSplitMetaInfo(new JobID(), fs, conf,
            submitDir);
    assertEquals("unexpected number of splits", 1, infos.length);
    assertEquals("unexpected number of split locations",
        4, infos[0].getLocations().length);
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 
Example #7
Source File: TestReporter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void map(LongWritable key, Text value, Context context)
    throws IOException {
  StringBuilder sb = new StringBuilder(512);
  for (int i = 0; i < 1000; i++) {
    sb.append("a");
  }
  context.setStatus(sb.toString());
  int progressStatusLength = context.getConfiguration().getInt(
      MRConfig.PROGRESS_STATUS_LEN_LIMIT_KEY,
      MRConfig.PROGRESS_STATUS_LEN_LIMIT_DEFAULT);

  if (context.getStatus().length() > progressStatusLength) {
    throw new IOException("Status is not truncated");
  }
}
 
Example #8
Source File: TestJobSplitWriter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMaxBlockLocationsNewSplits() throws Exception {
  TEST_DIR.mkdirs();
  try {
    Configuration conf = new Configuration();
    conf.setInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY, 4);
    Path submitDir = new Path(TEST_DIR.getAbsolutePath());
    FileSystem fs = FileSystem.getLocal(conf);
    FileSplit split = new FileSplit(new Path("/some/path"), 0, 1,
        new String[] { "loc1", "loc2", "loc3", "loc4", "loc5" });
    JobSplitWriter.createSplitFiles(submitDir, conf, fs,
        new FileSplit[] { split });
    JobSplit.TaskSplitMetaInfo[] infos =
        SplitMetaInfoReader.readSplitMetaInfo(new JobID(), fs, conf,
            submitDir);
    assertEquals("unexpected number of splits", 1, infos.length);
    assertEquals("unexpected number of split locations",
        4, infos[0].getLocations().length);
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 
Example #9
Source File: TestJobAclsManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testClusterAdmins() {
  Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
  Configuration conf = new Configuration();
  String jobOwner = "testuser";
  conf.set(JobACL.VIEW_JOB.getAclName(), jobOwner);
  conf.set(JobACL.MODIFY_JOB.getAclName(), jobOwner);
  conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
  String clusterAdmin = "testuser2";
  conf.set(MRConfig.MR_ADMINS, clusterAdmin);

  JobACLsManager aclsManager = new JobACLsManager(conf);
  tmpJobACLs = aclsManager.constructJobACLs(conf);
  final Map<JobACL, AccessControlList> jobACLs = tmpJobACLs;

  UserGroupInformation callerUGI = UserGroupInformation.createUserForTesting(
      clusterAdmin, new String[] {});

  // cluster admin should have access
  boolean val = aclsManager.checkAccess(callerUGI, JobACL.VIEW_JOB, jobOwner,
      jobACLs.get(JobACL.VIEW_JOB));
  assertTrue("cluster admin should have view access", val);
  val = aclsManager.checkAccess(callerUGI, JobACL.MODIFY_JOB, jobOwner,
      jobACLs.get(JobACL.MODIFY_JOB));
  assertTrue("cluster admin should have modify access", val);
}
 
Example #10
Source File: TestMRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example #11
Source File: TestJobClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetClusterStatusWithLocalJobRunner() throws Exception {
  Configuration conf = new Configuration();
  conf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  JobClient client = new JobClient(conf);
  ClusterStatus clusterStatus = client.getClusterStatus(true);
  Collection<String> activeTrackerNames = clusterStatus
      .getActiveTrackerNames();
  Assert.assertEquals(0, activeTrackerNames.size());
  int blacklistedTrackers = clusterStatus.getBlacklistedTrackers();
  Assert.assertEquals(0, blacklistedTrackers);
  Collection<BlackListInfo> blackListedTrackersInfo = clusterStatus
      .getBlackListedTrackersInfo();
  Assert.assertEquals(0, blackListedTrackersInfo.size());
}
 
Example #12
Source File: TestMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test 
public void testGetMasterUser() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MRConfig.MASTER_USER_NAME, "foo");
  conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");

  // default is yarn framework  
  assertEquals(Master.getMasterUserName(conf), "bar");

  // set framework name to classic
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertEquals(Master.getMasterUserName(conf), "foo");

  // change framework to yarn
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertEquals(Master.getMasterUserName(conf), "bar");

}
 
Example #13
Source File: TestJobAclsManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testClusterAdmins() {
  Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
  Configuration conf = new Configuration();
  String jobOwner = "testuser";
  conf.set(JobACL.VIEW_JOB.getAclName(), jobOwner);
  conf.set(JobACL.MODIFY_JOB.getAclName(), jobOwner);
  conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
  String clusterAdmin = "testuser2";
  conf.set(MRConfig.MR_ADMINS, clusterAdmin);

  JobACLsManager aclsManager = new JobACLsManager(conf);
  tmpJobACLs = aclsManager.constructJobACLs(conf);
  final Map<JobACL, AccessControlList> jobACLs = tmpJobACLs;

  UserGroupInformation callerUGI = UserGroupInformation.createUserForTesting(
      clusterAdmin, new String[] {});

  // cluster admin should have access
  boolean val = aclsManager.checkAccess(callerUGI, JobACL.VIEW_JOB, jobOwner,
      jobACLs.get(JobACL.VIEW_JOB));
  assertTrue("cluster admin should have view access", val);
  val = aclsManager.checkAccess(callerUGI, JobACL.MODIFY_JOB, jobOwner,
      jobACLs.get(JobACL.MODIFY_JOB));
  assertTrue("cluster admin should have modify access", val);
}
 
Example #14
Source File: TestMRFramework.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFramework() {
  JobConf jobConf = new JobConf();
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME);
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertFalse("Expected 'isLocal' to be false", 
      StreamUtil.isLocalJobTracker(jobConf));
  
  jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090");
  jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
  assertTrue("Expected 'isLocal' to be true", 
      StreamUtil.isLocalJobTracker(jobConf));
}
 
Example #15
Source File: IFileInputStream.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Create a checksum input stream that reads
 * @param in The input stream to be verified for checksum.
 * @param len The length of the input stream including checksum bytes.
 */
public IFileInputStream(InputStream in, long len, Configuration conf) {
  this.in = in;
  this.inFd = getFileDescriptorIfAvail(in);
  sum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 
      Integer.MAX_VALUE);
  checksumSize = sum.getChecksumSize();
  length = len;
  dataLength = length - checksumSize;

  conf = (conf != null) ? conf : new Configuration();
  readahead = conf.getBoolean(MRConfig.MAPRED_IFILE_READAHEAD,
      MRConfig.DEFAULT_MAPRED_IFILE_READAHEAD);
  readaheadLength = conf.getInt(MRConfig.MAPRED_IFILE_READAHEAD_BYTES,
      MRConfig.DEFAULT_MAPRED_IFILE_READAHEAD_BYTES);

  doReadahead();
}
 
Example #16
Source File: TestBinaryTokenFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  final Configuration conf = new Configuration();
  
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" + SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG");
  
  final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
  builder.checkExitOnShutdown(true);
  builder.numDataNodes(numSlaves);
  builder.format(true);
  builder.racks(null);
  dfsCluster = builder.build();
  
  mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs);
  mrCluster.init(conf);
  mrCluster.start();

  NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); 
  
  FileSystem fs = dfsCluster.getFileSystem(); 
  p1 = new Path("file1");
  p1 = fs.makeQualified(p1);
}
 
Example #17
Source File: Task.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile = ReflectionUtils.newInstance(
      conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
        MROutputFiles.class, MapOutputFile.class), conf);
  this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 
Example #18
Source File: TestMRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example #19
Source File: TestMRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example #20
Source File: Task.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static String normalizeStatus(String status, Configuration conf) {
  // Check to see if the status string is too long
  // and truncate it if needed.
  int progressStatusLength = conf.getInt(
      MRConfig.PROGRESS_STATUS_LEN_LIMIT_KEY,
      MRConfig.PROGRESS_STATUS_LEN_LIMIT_DEFAULT);
  if (status.length() > progressStatusLength) {
    LOG.warn("Task status: \"" + status + "\" truncated to max limit ("
        + progressStatusLength + " characters)");
    status = status.substring(0, progressStatusLength);
  }
  return status;
}
 
Example #21
Source File: JobSplitWriter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T extends InputSplit> 
SplitMetaInfo[] writeNewSplits(Configuration conf, 
    T[] array, FSDataOutputStream out)
throws IOException, InterruptedException {

  SplitMetaInfo[] info = new SplitMetaInfo[array.length];
  if (array.length != 0) {
    SerializationFactory factory = new SerializationFactory(conf);
    int i = 0;
    int maxBlockLocations = conf.getInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY,
        MRConfig.MAX_BLOCK_LOCATIONS_DEFAULT);
    long offset = out.getPos();
    for(T split: array) {
      long prevCount = out.getPos();
      Text.writeString(out, split.getClass().getName());
      Serializer<T> serializer = 
        factory.getSerializer((Class<T>) split.getClass());
      serializer.open(out);
      serializer.serialize(split);
      long currCount = out.getPos();
      String[] locations = split.getLocations();
      if (locations.length > maxBlockLocations) {
        LOG.warn("Max block location exceeded for split: "
            + split + " splitsize: " + locations.length +
            " maxsize: " + maxBlockLocations);
        locations = Arrays.copyOf(locations, maxBlockLocations);
      }
      info[i++] = 
        new JobSplit.SplitMetaInfo( 
            locations, offset,
            split.getLength());
      offset += currCount - prevCount;
    }
  }
  return info;
}
 
Example #22
Source File: TestMapProcessor.java    From tez with Apache License 2.0 5 votes vote down vote up
public void setUpJobConf(JobConf job) {
  job.set(TezRuntimeFrameworkConfigs.LOCAL_DIRS, workDir.toString());
  job.set(MRConfig.LOCAL_DIR, workDir.toString());
  job.setClass(
      Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
      TezTaskOutputFiles.class,
      TezTaskOutput.class);
  job.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
  job.setNumReduceTasks(1);
}
 
Example #23
Source File: Master.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static String getMasterUserName(Configuration conf) {
  String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {    
    return conf.get(MRConfig.MASTER_USER_NAME);
  } 
  else {
    return conf.get(YarnConfiguration.RM_PRINCIPAL);
  }
}
 
Example #24
Source File: QueueManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/***
 * Dumps the configuration of hierarchy of queues with 
 * the xml file path given. It is to be used directly ONLY FOR TESTING.
 * @param out the writer object to which dump is written to.
 * @param configFile the filename of xml file
 * @throws IOException
 */
static void dumpConfiguration(Writer out, String configFile,
    Configuration conf) throws IOException {
  if (conf != null && conf.get(DeprecatedQueueConfigurationParser.
      MAPRED_QUEUE_NAMES_KEY) != null) {
    return;
  }
  
  JsonFactory dumpFactory = new JsonFactory();
  JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
  QueueConfigurationParser parser;
  boolean aclsEnabled = false;
  if (conf != null) {
    aclsEnabled = conf.getBoolean(MRConfig.MR_ACLS_ENABLED, false);
  }
  if (configFile != null && !"".equals(configFile)) {
    parser = new QueueConfigurationParser(configFile, aclsEnabled);
  }
  else {
    parser = getQueueConfigurationParser(null, false, aclsEnabled);
  }
  dumpGenerator.writeStartObject();
  dumpGenerator.writeFieldName("queues");
  dumpGenerator.writeStartArray();
  dumpConfiguration(dumpGenerator,parser.getRoot().getChildren());
  dumpGenerator.writeEndArray();
  dumpGenerator.writeEndObject();
  dumpGenerator.flush();
}
 
Example #25
Source File: ShuffleHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public HttpPipelineFactory(Configuration conf) throws Exception {
  SHUFFLE = getShuffle(conf);
  if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
                      MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
    LOG.info("Encrypted shuffle is enabled.");
    sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
    sslFactory.init();
  }
}
 
Example #26
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 3000000)
public void testSetClasspathWithFramework() throws IOException {
  final String FRAMEWORK_NAME = "some-framework-name";
  final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
    fail("Failed to catch framework path set without classpath change");
  } catch (IllegalArgumentException e) {
    assertTrue("Unexpected IllegalArgumentException",
        e.getMessage().contains("Could not locate MapReduce framework name '"
            + FRAMEWORK_NAME + "'"));
  }

  env.clear();
  final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
  MRApps.setClasspath(env, conf);
  final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
          ApplicationConstants.Environment.PWD.$$() + "/*"));
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          FRAMEWORK_CLASSPATH, stdClasspath));
  assertEquals("Incorrect classpath with framework and no user precedence",
      expectedClasspath, env.get("CLASSPATH"));

  env.clear();
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
  MRApps.setClasspath(env, conf);
  expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          stdClasspath, FRAMEWORK_CLASSPATH));
  assertEquals("Incorrect classpath with framework and user precedence",
      expectedClasspath, env.get("CLASSPATH"));
}
 
Example #27
Source File: IgniteHadoopClientProtocolProvider.java    From ignite with Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
@Override public ClientProtocol create(Configuration conf) throws IOException {
    if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
        Collection<String> addrs = conf.getTrimmedStringCollection(MRConfig.MASTER_ADDRESS);

        if (F.isEmpty(addrs))
            throw new IOException("Failed to create client protocol because Ignite node addresses are not " +
                "specified (did you set " + MRConfig.MASTER_ADDRESS + " property?).");

        if (F.contains(addrs, "local"))
            throw new IOException("Local execution mode is not supported, please point " +
                MRConfig.MASTER_ADDRESS + " to real Ignite nodes.");

        Collection<String> addrs0 = new ArrayList<>(addrs.size());

        // Set up port by default if need
        for (String addr : addrs) {
            if (!addr.contains(":"))
                addrs0.add(addr + ':' + ConnectorConfiguration.DFLT_TCP_PORT);
            else
                addrs0.add(addr);
        }

        return new HadoopClientProtocol(conf, client(conf.get(MRConfig.MASTER_ADDRESS), addrs0));
    }

    return null;
}
 
Example #28
Source File: TestMapReduceChildJVM.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testCommandLine() throws Exception {

  MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true);
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = app.submit(conf);
  app.waitForState(job, JobState.SUCCEEDED);
  app.verifyCompleted();

  Assert.assertEquals(
    "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
    " -Djava.net.preferIPv4Stack=true" +
    " -Dhadoop.metrics.log.level=WARN" +
    "  -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
    " -Dlog4j.configuration=container-log4j.properties" +
    " -Dyarn.app.container.log.dir=<LOG_DIR>" +
    " -Dyarn.app.container.log.filesize=0" +
    " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
    " org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
    " 54321" +
    " attempt_0_0000_m_000000_0" +
    " 0" +
    " 1><LOG_DIR>/stdout" +
    " 2><LOG_DIR>/stderr ]", app.myCommandLine);
  
  Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
    app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
  Assert.assertEquals("INFO,console",
    app.cmdEnvironment.get("HADOOP_ROOT_LOGGER"));
  Assert.assertTrue("HADOOP_CLIENT_OPTS not set for job",
    app.cmdEnvironment.containsKey("HADOOP_CLIENT_OPTS"));
  Assert.assertEquals("", app.cmdEnvironment.get("HADOOP_CLIENT_OPTS"));
}
 
Example #29
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$()
        + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String yarnAppClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (yarnAppClasspath != null) {
    yarnAppClasspath =
        yarnAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(
          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
          MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath =
        mrAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
 
Example #30
Source File: TestHighRamJob.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void testHighRamConfig(long jobMapMB, long jobReduceMB, 
    long clusterMapMB, long clusterReduceMB, long simulatedClusterMapMB, 
    long simulatedClusterReduceMB, long expectedMapMB, long expectedReduceMB, 
    Configuration gConf) 
throws IOException {
  Configuration simulatedJobConf = new Configuration(gConf);
  simulatedJobConf.setLong(MRConfig.MAPMEMORY_MB, simulatedClusterMapMB);
  simulatedJobConf.setLong(MRConfig.REDUCEMEMORY_MB, 
                           simulatedClusterReduceMB);
  
  // define a source conf
  Configuration sourceConf = new Configuration();
  
  // configure the original job
  sourceConf.setLong(MRJobConfig.MAP_MEMORY_MB, jobMapMB);
  sourceConf.setLong(MRConfig.MAPMEMORY_MB, clusterMapMB);
  sourceConf.setLong(MRJobConfig.REDUCE_MEMORY_MB, jobReduceMB);
  sourceConf.setLong(MRConfig.REDUCEMEMORY_MB, clusterReduceMB);
  
  // define a mock job
  MockJob story = new MockJob(sourceConf);
  
  GridmixJob job = new DummyGridmixJob(simulatedJobConf, story);
  Job simulatedJob = job.getJob();
  Configuration simulatedConf = simulatedJob.getConfiguration();
  
  // check if the high ram properties are not set
  assertEquals(expectedMapMB, 
               simulatedConf.getLong(MRJobConfig.MAP_MEMORY_MB,
                                     MRJobConfig.DEFAULT_MAP_MEMORY_MB));
  assertEquals(expectedReduceMB, 
               simulatedConf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 
                                     MRJobConfig.DEFAULT_MAP_MEMORY_MB));
}