Java Code Examples for org.apache.hadoop.mapred.JobConf#setFloat()

The following examples show how to use org.apache.hadoop.mapred.JobConf#setFloat() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ItemModelTest.java From ml-ease with Apache License 2.0

6 votes

@Override
public void run() throws Exception
{
  JobConfig props = super.getJobConfig();
  List<String> lambdastr = props.getStringList(LAMBDA, ",");
  String outBasePath = props.getString(OUTPUT_BASE_PATH);
  for (String lambda : lambdastr)
  {
    String outPath = outBasePath + "/lambda-" + lambda;
    props.put("output.path", outPath);
    JobConf conf = createJobConf(PerItemTestMapper.class, PerItemTestReducer.class);
    AvroUtils.addAvroCacheFilesAndSetTheProperty(conf,
                                                 new Path(props.get(MODEL_PATH)),
                                                 MODEL_PATH);
    conf.set(ITEM_KEY, props.getString(ITEM_KEY));
    conf.setFloat(LAMBDA, Float.parseFloat(lambda));
    conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false));
    conf.setPartitionerClass(PerItemTestPartitioner.class);
    conf.setInt(NUM_REDUCERS, conf.getNumReduceTasks());
    AvroUtils.runAvroJob(conf);
  }
}

Example 2

Source File: DistCp.java From RDFS with Apache License 2.0

6 votes

private static JobConf createJobConf(Configuration conf, boolean useFastCopy) {
  Class<? extends InputFormat> inputFormat =
    (useFastCopy) ? FastCopyInputFormat.class : CopyInputFormat.class;
  JobConf jobconf = new JobConf(conf, DistCp.class);
  jobconf.setJobName(NAME);

  // turn off speculative execution, because DFS doesn't handle
  // multiple writers to the same file.
  jobconf.setReduceSpeculativeExecution(false);
  jobconf.setMapOutputKeyClass(FilePairComparable.class);
  jobconf.setMapOutputValueClass(Text.class);
  jobconf.setOutputKeyClass(FilePairComparable.class);
  jobconf.setOutputValueClass(Text.class);

  jobconf.setInputFormat(inputFormat);
  jobconf.setMapperClass(CopyFilesTask.class);
  jobconf.setReducerClass(CopyFilesTask.class);
    
  jobconf.setNumReduceTasks(conf.getInt(MAX_REDUCE_LABEL, 1));
  // Prevent the reducer from starting until all maps are done.
  jobconf.setInt("mapred.job.rushreduce.reduce.threshold", 0);
  jobconf.setFloat("mapred.reduce.slowstart.completed.maps", 1.0f);
  return jobconf;
}

Example 3

Source File: TestDeprecatedKeys.java From incubator-tez with Apache License 2.0

5 votes

@Test
public void verifyReduceKeyTranslation() {
  JobConf jobConf = new JobConf();

  jobConf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.4f);
  jobConf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, 20000l);
  jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, 2000);
  jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.55f);
  jobConf.setFloat(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, 0.60f);
  jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
  jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
  jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);

  MRHelpers.translateVertexConfToTez(jobConf);

  assertEquals(0.4f, jobConf.getFloat(
      TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0f), 0.01f);
  assertEquals(20000l, jobConf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
  assertEquals(2000,
      jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(0.55f, jobConf.getFloat(
      TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0), 0.01f);
  assertEquals(0.60f,
      jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0),
      0.01f);
  assertEquals(0.22f,
      jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0),
      0.01f);
  assertEquals(true, jobConf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
  assertEquals(0.33f,
      jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0),
      0.01f);
}

Example 4

Source File: TestDeprecatedKeys.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void verifyReduceKeyTranslation() {
  JobConf jobConf = new JobConf();

  jobConf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.4f);
  jobConf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, 20000l);
  jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, 2000);
  jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.55f);
  jobConf.setFloat(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, 0.60f);
  jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
  jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
  jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);
  jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);

  MRHelpers.translateMRConfToTez(jobConf);

  assertEquals(0.4f, jobConf.getFloat(
      TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0f), 0.01f);
  assertEquals(20000l, jobConf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
  assertEquals(2000,
      jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(0.55f, jobConf.getFloat(
      TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0), 0.01f);
  assertEquals(0.60f,
      jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0),
      0.01f);
  assertEquals(0.22f,
      jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0),
      0.01f);
  assertEquals(true, jobConf.getBoolean(
      TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
  assertEquals(0.33f,
      jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0),
      0.01f);
  assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));
}

Example 5

Source File: TestMergeManager.java From hadoop with Apache License 2.0

4 votes

@Test(timeout=10000)
public void testMemoryMerge() throws Exception {
  final int TOTAL_MEM_BYTES = 10000;
  final int OUTPUT_SIZE = 7950;
  JobConf conf = new JobConf();
  conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f);
  conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, TOTAL_MEM_BYTES);
  conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.8f);
  conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.9f);
  TestExceptionReporter reporter = new TestExceptionReporter();
  CyclicBarrier mergeStart = new CyclicBarrier(2);
  CyclicBarrier mergeComplete = new CyclicBarrier(2);
  StubbedMergeManager mgr = new StubbedMergeManager(conf, reporter,
      mergeStart, mergeComplete);

  // reserve enough map output to cause a merge when it is committed
  MapOutput<Text, Text> out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                    (out1 instanceof InMemoryMapOutput));
  InMemoryMapOutput<Text, Text> mout1 = (InMemoryMapOutput<Text, Text>)out1;
  fillOutput(mout1);
  MapOutput<Text, Text> out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                    (out2 instanceof InMemoryMapOutput));
  InMemoryMapOutput<Text, Text> mout2 = (InMemoryMapOutput<Text, Text>)out2;
  fillOutput(mout2);

  // next reservation should be a WAIT
  MapOutput<Text, Text> out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertEquals("Should be told to wait", null, out3);

  // trigger the first merge and wait for merge thread to start merging
  // and free enough output to reserve more
  mout1.commit();
  mout2.commit();
  mergeStart.await();

  Assert.assertEquals(1, mgr.getNumMerges());

  // reserve enough map output to cause another merge when committed
  out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                     (out1 instanceof InMemoryMapOutput));
  mout1 = (InMemoryMapOutput<Text, Text>)out1;
  fillOutput(mout1);
  out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                     (out2 instanceof InMemoryMapOutput));
  mout2 = (InMemoryMapOutput<Text, Text>)out2;
  fillOutput(mout2);

  // next reservation should be null
  out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertEquals("Should be told to wait", null, out3);

  // commit output *before* merge thread completes
  mout1.commit();
  mout2.commit();

  // allow the first merge to complete
  mergeComplete.await();

  // start the second merge and verify
  mergeStart.await();
  Assert.assertEquals(2, mgr.getNumMerges());

  // trigger the end of the second merge
  mergeComplete.await();

  Assert.assertEquals(2, mgr.getNumMerges());
  Assert.assertEquals("exception reporter invoked",
      0, reporter.getNumExceptions());
}

Example 6

Source File: TestMergeManager.java From big-c with Apache License 2.0

4 votes

@Test(timeout=10000)
public void testMemoryMerge() throws Exception {
  final int TOTAL_MEM_BYTES = 10000;
  final int OUTPUT_SIZE = 7950;
  JobConf conf = new JobConf();
  conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f);
  conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, TOTAL_MEM_BYTES);
  conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.8f);
  conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.9f);
  TestExceptionReporter reporter = new TestExceptionReporter();
  CyclicBarrier mergeStart = new CyclicBarrier(2);
  CyclicBarrier mergeComplete = new CyclicBarrier(2);
  StubbedMergeManager mgr = new StubbedMergeManager(conf, reporter,
      mergeStart, mergeComplete);

  // reserve enough map output to cause a merge when it is committed
  MapOutput<Text, Text> out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                    (out1 instanceof InMemoryMapOutput));
  InMemoryMapOutput<Text, Text> mout1 = (InMemoryMapOutput<Text, Text>)out1;
  fillOutput(mout1);
  MapOutput<Text, Text> out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                    (out2 instanceof InMemoryMapOutput));
  InMemoryMapOutput<Text, Text> mout2 = (InMemoryMapOutput<Text, Text>)out2;
  fillOutput(mout2);

  // next reservation should be a WAIT
  MapOutput<Text, Text> out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertEquals("Should be told to wait", null, out3);

  // trigger the first merge and wait for merge thread to start merging
  // and free enough output to reserve more
  mout1.commit();
  mout2.commit();
  mergeStart.await();

  Assert.assertEquals(1, mgr.getNumMerges());

  // reserve enough map output to cause another merge when committed
  out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                     (out1 instanceof InMemoryMapOutput));
  mout1 = (InMemoryMapOutput<Text, Text>)out1;
  fillOutput(mout1);
  out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertTrue("Should be a memory merge",
                     (out2 instanceof InMemoryMapOutput));
  mout2 = (InMemoryMapOutput<Text, Text>)out2;
  fillOutput(mout2);

  // next reservation should be null
  out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
  Assert.assertEquals("Should be told to wait", null, out3);

  // commit output *before* merge thread completes
  mout1.commit();
  mout2.commit();

  // allow the first merge to complete
  mergeComplete.await();

  // start the second merge and verify
  mergeStart.await();
  Assert.assertEquals(2, mgr.getNumMerges());

  // trigger the end of the second merge
  mergeComplete.await();

  Assert.assertEquals(2, mgr.getNumMerges());
  Assert.assertEquals("exception reporter invoked",
      0, reporter.getNumExceptions());
}

Example 7

Source File: ItemModelTrain.java From ml-ease with Apache License 2.0

4 votes

@Override
public void run() throws Exception
{
  JobConfig props = super.getJobConfig();
  _logger.info("Start training per-key naive logistic regression model...");
  String outBasePath = props.getString(OUTPUT_MODEL_PATH);
  String outpath = outBasePath + "/models";
  props.put("output.path", outpath);
  JobConf conf =
      createJobConf(ItemModelTrainMapper.class,
                    ItemModelTrainReducer.class,
                    Pair.getPairSchema(Schema.create(Type.STRING),
                                       RegressionPrepareOutput.SCHEMA$),
                    LinearModelWithVarAvro.SCHEMA$);
  // set up conf
  String interceptPriorMeanMap = props.getString(INTERCEPT_PRIOR_MEAN_MAP,"");
  if (!interceptPriorMeanMap.equals(""))
  {
    AvroUtils.addAvroCacheFilesAndSetTheProperty(conf, new Path(interceptPriorMeanMap), INTERCEPT_PRIOR_MEAN_MAP);
  }
  String lambdaMap = props.getString(LAMBDA_MAP,"");
  if (!lambdaMap.equals(""))
  {
    AvroUtils.addAvroCacheFilesAndSetTheProperty(conf, new Path(lambdaMap), LAMBDA_MAP);
  }
  conf.setFloat(INTERCEPT_DEFAULT_PRIOR_MEAN, (float)props.getDouble(INTERCEPT_DEFAULT_PRIOR_MEAN,0));
  conf.set(INTERCEPT_LAMBDAS,props.get(INTERCEPT_LAMBDAS));
  conf.set(DEFAULT_LAMBDAS,props.get(DEFAULT_LAMBDAS));
  conf.setLong(REPORT_FREQUENCY, props.getLong(REPORT_FREQUENCY, 1000000));
  conf.setFloat(LIBLINEAR_EPSILON, (float) props.getDouble(LIBLINEAR_EPSILON, 0.001f));
  conf.setBoolean(COMPUTE_VAR, props.getBoolean(COMPUTE_VAR,false));
  conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false));
  conf.setBoolean(SHORT_FEATURE_INDEX, props.getBoolean(SHORT_FEATURE_INDEX, false));
  // run job
  AvroUtils.runAvroJob(conf);
  boolean removeTmpDir = props.getBoolean(REMOVE_TMP_DIR, true);
  if (removeTmpDir)
  {
    FileSystem fs = FileSystem.get(conf);
    fs.delete(new Path(outBasePath + "/tmp-data"), true);
  }
}

Example 8

Source File: TestDeprecatedKeys.java From incubator-tez with Apache License 2.0

4 votes

@Test
/**
 * Set of keys that can be overriden at tez runtime
 */
public void verifyTezOverridenKeys() {
  JobConf jobConf = new JobConf();
  jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, 2000);
  jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
  jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
  
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 200);
  jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 20);
  jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.2f);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 10);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 20);
  jobConf.setInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 10);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 10);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 10);
  jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, true);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 10);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 10);
  jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
  jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 10.0f);
  jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 10.0f);
  jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 10.0f);
  jobConf.setInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 10);
  jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
  jobConf.setFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 10.0f);
  jobConf.set(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
  jobConf.set(TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
  jobConf.set(TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
  
  jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
  jobConf.setBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, true);

  MRHelpers.translateVertexConfToTez(jobConf);

  assertEquals(1000, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(200, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 100));
  assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false));
  assertEquals(20, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
  assertEquals(20, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 0));
  assertEquals(10, jobConf.getInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 0));
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 0));
  assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, false));
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 0));
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 0));
  assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, false));
  assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
  assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.0f);
  assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.0f);
  assertEquals(10, jobConf.getInt(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0));
  assertEquals(true, jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
  assertEquals(10.0f, jobConf.getFloat(TezJobConfig.TEZ_RUNTIME_INPUT_BUFFER_PERCENT, 0.0f), 0.0f);
  assertEquals("DefaultSorter", jobConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
  assertEquals("groupComparator", jobConf.get(TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, ""));
  assertEquals("SecondaryComparator", jobConf.get(TezJobConfig.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, ""));
  assertEquals("DefaultSorter", jobConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
  assertTrue(jobConf.getBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, false));

  assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
  assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
  assertNull(jobConf.get(MRJobConfig.RECORDS_BEFORE_PROGRESS));
  assertNull(jobConf.get(MRJobConfig.IO_SORT_FACTOR));
  assertNull(jobConf.get(MRJobConfig.IO_SORT_MB));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_READ_TIMEOUT));
  assertNull(jobConf.get(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT));
  assertNull(jobConf.get(MRJobConfig.MAP_COMBINE_MIN_SPILLS));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_PARALLEL_COPIES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_FETCH_FAILURES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_NOTIFY_READERROR));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_READ_TIMEOUT));
  assertNull(jobConf.get(MRConfig.SHUFFLE_SSL_ENABLED_KEY));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMTOMEM_ENABLED));
  assertNull(jobConf.get(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT));
  assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
  assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
  assertNull(jobConf.get("map.sort.class"));
}

Example 9

Source File: TestDeprecatedKeys.java From tez with Apache License 2.0

4 votes

@Test(timeout = 5000)
/**
 * Set of keys that can be overridden at tez runtime
 */
public void verifyTezOverridenKeys() {
  JobConf jobConf = new JobConf();
  jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, 2000);
  jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
  jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
  jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f);
  jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
  jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 20);
  jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.2f);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 10);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 20);
  jobConf.setInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 10);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 10);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 10);
  jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, true);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 10);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 10);
  jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
  jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 10.0f);
  jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 10.0f);
  jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 10.0f);
  jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 10);
  jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, true);
  jobConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 10.0f);
  jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
  jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
  jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
  jobConf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);

  jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
  jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);

  MRHelpers.translateMRConfToTez(jobConf);

  assertEquals(1000, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0));
  assertEquals(200, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 100));
  assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false));
  assertEquals(20, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0));
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0));
  assertEquals(20, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 0));
  assertEquals(10, jobConf.getInt(Constants.TEZ_RUNTIME_TASK_MEMORY, 0));
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 0));
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 0));
  assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, false));
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 0));
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 0));
  assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, false));
  assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.0f);
  assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.0f);
  assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.0f);
  assertEquals(10, jobConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, 0));
  assertEquals(true, jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, false));
  assertEquals(10.0f, jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.0f), 0.0f);
  assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
  assertEquals("groupComparator", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, ""));
  assertEquals("SecondaryComparator", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, ""));
  assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
  assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
  assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f);
  assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));

  assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
  assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
  assertNull(jobConf.get(MRJobConfig.RECORDS_BEFORE_PROGRESS));
  assertNull(jobConf.get(MRJobConfig.IO_SORT_FACTOR));
  assertNull(jobConf.get(MRJobConfig.IO_SORT_MB));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_READ_TIMEOUT));
  assertNull(jobConf.get(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT));
  assertNull(jobConf.get(MRJobConfig.MAP_COMBINE_MIN_SPILLS));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_PARALLEL_COPIES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_FETCH_FAILURES));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_NOTIFY_READERROR));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_READ_TIMEOUT));
  assertNull(jobConf.get(MRConfig.SHUFFLE_SSL_ENABLED_KEY));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT));
  assertNull(jobConf.get(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD));
  assertNull(jobConf.get(MRJobConfig.REDUCE_MEMTOMEM_ENABLED));
  assertNull(jobConf.get(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT));
  assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
  assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
  assertNull(jobConf.get("map.sort.class"));
  assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART));
  assertNull(jobConf.get(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST));
}