Java Code Examples for org.apache.hadoop.mapreduce.TaskInputOutputContext#getConfiguration()

The following examples show how to use org.apache.hadoop.mapreduce.TaskInputOutputContext#getConfiguration() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LoadJob.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
ResourceUsageMatcherRunner(final TaskInputOutputContext context, 
                           ResourceUsageMetrics metrics) {
  Configuration conf = context.getConfiguration();
  
  // set the resource calculator plugin
  Class<? extends ResourceCalculatorPlugin> clazz =
    conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
                  null, ResourceCalculatorPlugin.class);
  ResourceCalculatorPlugin plugin = 
    ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
  
  // set the other parameters
  this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
  progress = new BoostingProgress(context);
  
  // instantiate a resource-usage-matcher
  matcher = new ResourceUsageMatcher();
  matcher.configure(conf, plugin, metrics, progress);
}
Example 2
Source File: LoadJob.java    From big-c with Apache License 2.0 6 votes vote down vote up 
ResourceUsageMatcherRunner(final TaskInputOutputContext context, 
                           ResourceUsageMetrics metrics) {
  Configuration conf = context.getConfiguration();
  
  // set the resource calculator plugin
  Class<? extends ResourceCalculatorPlugin> clazz =
    conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
                  null, ResourceCalculatorPlugin.class);
  ResourceCalculatorPlugin plugin = 
    ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
  
  // set the other parameters
  this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
  progress = new BoostingProgress(context);
  
  // instantiate a resource-usage-matcher
  matcher = new ResourceUsageMatcher();
  matcher.configure(conf, plugin, metrics, progress);
}
Example 3
Source File: MapReducePOStoreImpl.java    From spork with Apache License 2.0 6 votes vote down vote up 
public MapReducePOStoreImpl(TaskInputOutputContext<?,?,?,?> context) {
    // get a copy of the Configuration so that changes to the
    // configuration below (like setting the output location) do
    // not affect the caller's copy
    Configuration outputConf = new Configuration(context.getConfiguration());
    reporter = PigStatusReporter.getInstance();
    reporter.setContext(new MRTaskContext(context));

    // make a copy of the Context to use here - since in the same
    // task (map or reduce) we could have multiple stores, we should
    // make this copy so that the same context does not get over-written
    // by the different stores.

    this.context = HadoopShims.createTaskAttemptContext(outputConf,
            context.getTaskAttemptID());
}
Example 4
Source File: MetricsService.java    From datawave with Apache License 2.0 5 votes vote down vote up 
public MetricsService(ContextWriter<OK,OV> contextWriter, TaskInputOutputContext<?,?,OK,OV> context) {
    Configuration conf = context.getConfiguration();
    
    this.date = DateHelper.format(new Date());
    
    this.fieldNames = MetricsConfiguration.getFieldNames(conf);
    this.enabledLabels = MetricsConfiguration.getLabels(conf);
    this.enabledKeys = enabledLabels.keySet();
    
    this.wildcardedLabels = new HashSet<>();
    for (Map.Entry<String,String> entry : enabledLabels.entries()) {
        if (WILDCARD.equals(entry.getValue())) {
            wildcardedLabels.add(entry.getKey());
        }
    }
    
    this.receivers = new HashMap<>();
    for (MetricsReceiver receiver : MetricsConfiguration.getReceivers(conf)) {
        this.receivers.put(receiver.getMetric(), receiver);
        receiver.configure(conf, date);
    }
    
    this.store = new AggregatingMetricsStore<>(contextWriter, context);
    
    if (logger.isInfoEnabled()) {
        logger.info("Metrics Service Initialized");
        logger.info("enabledLabels = " + enabledLabels);
        logger.info("receivers = " + receivers);
        logger.info("fieldNames = " + fieldNames);
    }
    
    Preconditions.checkNotNull(fieldNames);
    Preconditions.checkArgument(!enabledLabels.isEmpty());
    Preconditions.checkArgument(!receivers.isEmpty());
}
Example 5
Source File: TestMRWithDistributedCache.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void setup(TaskInputOutputContext<?, ?, ?, ?> context)
    throws IOException {
  Configuration conf = context.getConfiguration();
  Path[] localFiles = context.getLocalCacheFiles();
  URI[] files = context.getCacheFiles();
  Path[] localArchives = context.getLocalCacheArchives();
  URI[] archives = context.getCacheArchives();
  FileSystem fs = LocalFileSystem.get(conf);

  // Check that 2 files and 2 archives are present
  TestCase.assertEquals(2, localFiles.length);
  TestCase.assertEquals(2, localArchives.length);
  TestCase.assertEquals(2, files.length);
  TestCase.assertEquals(2, archives.length);

  // Check the file name
  TestCase.assertTrue(files[0].getPath().endsWith("distributed.first"));
  TestCase.assertTrue(files[1].getPath().endsWith("distributed.second.jar"));
  
  // Check lengths of the files
  TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen());
  TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1);

  // Check extraction of the archive
  TestCase.assertTrue(fs.exists(new Path(localArchives[0],
      "distributed.jar.inside3")));
  TestCase.assertTrue(fs.exists(new Path(localArchives[1],
      "distributed.jar.inside4")));

  // Check the class loaders
  LOG.info("Java Classpath: " + System.getProperty("java.class.path"));
  ClassLoader cl = Thread.currentThread().getContextClassLoader();
  // Both the file and the archive were added to classpath, so both
  // should be reachable via the class loader.
  TestCase.assertNotNull(cl.getResource("distributed.jar.inside2"));
  TestCase.assertNotNull(cl.getResource("distributed.jar.inside3"));
  TestCase.assertNull(cl.getResource("distributed.jar.inside4"));

  // Check that the symlink for the renaming was created in the cwd;
  TestCase.assertTrue("symlink distributed.first.symlink doesn't exist",
      symlinkFile.exists());
  TestCase.assertEquals("symlink distributed.first.symlink length not 1", 1,
      symlinkFile.length());
  
  //This last one is a difference between MRv2 and MRv1
  TestCase.assertTrue("second file should be symlinked too",
      expectedAbsentSymlinkFile.exists());
}
Example 6
Source File: TestMRWithDistributedCache.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void setup(TaskInputOutputContext<?, ?, ?, ?> context)
    throws IOException {
  Configuration conf = context.getConfiguration();
  Path[] localFiles = context.getLocalCacheFiles();
  URI[] files = context.getCacheFiles();
  Path[] localArchives = context.getLocalCacheArchives();
  URI[] archives = context.getCacheArchives();
  FileSystem fs = LocalFileSystem.get(conf);

  // Check that 2 files and 2 archives are present
  TestCase.assertEquals(2, localFiles.length);
  TestCase.assertEquals(2, localArchives.length);
  TestCase.assertEquals(2, files.length);
  TestCase.assertEquals(2, archives.length);

  // Check the file name
  TestCase.assertTrue(files[0].getPath().endsWith("distributed.first"));
  TestCase.assertTrue(files[1].getPath().endsWith("distributed.second.jar"));
  
  // Check lengths of the files
  TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen());
  TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1);

  // Check extraction of the archive
  TestCase.assertTrue(fs.exists(new Path(localArchives[0],
      "distributed.jar.inside3")));
  TestCase.assertTrue(fs.exists(new Path(localArchives[1],
      "distributed.jar.inside4")));

  // Check the class loaders
  LOG.info("Java Classpath: " + System.getProperty("java.class.path"));
  ClassLoader cl = Thread.currentThread().getContextClassLoader();
  // Both the file and the archive were added to classpath, so both
  // should be reachable via the class loader.
  TestCase.assertNotNull(cl.getResource("distributed.jar.inside2"));
  TestCase.assertNotNull(cl.getResource("distributed.jar.inside3"));
  TestCase.assertNull(cl.getResource("distributed.jar.inside4"));

  // Check that the symlink for the renaming was created in the cwd;
  TestCase.assertTrue("symlink distributed.first.symlink doesn't exist",
      symlinkFile.exists());
  TestCase.assertEquals("symlink distributed.first.symlink length not 1", 1,
      symlinkFile.length());
  
  //This last one is a difference between MRv2 and MRv1
  TestCase.assertTrue("second file should be symlinked too",
      expectedAbsentSymlinkFile.exists());
}