Java Code Examples for org.apache.hadoop.mapred.Reducer#reduce()

The following examples show how to use org.apache.hadoop.mapred.Reducer#reduce() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
Example 2
Source File: MergeManagerImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
Example 3
Source File: MRCombiner.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException {
  Class<? extends Reducer> reducerClazz = (Class<? extends Reducer>) conf.getClass("mapred.combiner.class", null, Reducer.class);
  
  Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf);
  
  OutputCollector collector = new OutputCollector() {
    @Override
    public void collect(Object key, Object value) throws IOException {
      writer.append(key, value);
    }
  };
  
  CombinerValuesIterator values = new CombinerValuesIterator(rawIter, keyClass, valClass, comparator);
  
  while (values.moveToNext()) {
    combiner.reduce(values.getKey(), values.getValues().iterator(), collector, reporter);
  }
}
Example 4
Source File: MRCombiner.java    From tez with Apache License 2.0 6 votes vote down vote up 
private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException {
  Class<? extends Reducer> reducerClazz = (Class<? extends Reducer>) conf.getClass("mapred.combiner.class", null, Reducer.class);
  
  Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf);
  
  OutputCollector collector = new OutputCollector() {
    @Override
    public void collect(Object key, Object value) throws IOException {
      writer.append(key, value);
      combineOutputRecordsCounter.increment(1);
    }
  };
  
  CombinerValuesIterator values = new CombinerValuesIterator(rawIter, keyClass, valClass, comparator);
  
  while (values.moveToNext()) {
    combiner.reduce(values.getKey(), values.getValues().iterator(), collector, reporter);
  }
}
Example 5
Source File: ReduceProcessor.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
void runOldReducer(JobConf job,
    final MRTaskReporter reporter,
    KeyValuesReader input,
    RawComparator comparator,
    Class keyClass,
    Class valueClass,
    final KeyValueWriter output) throws IOException, InterruptedException {

  Reducer reducer =
      ReflectionUtils.newInstance(job.getReducerClass(), job);

  // make output collector

  OutputCollector collector =
      new OutputCollector() {
    public void collect(Object key, Object value)
        throws IOException {
      output.write(key, value);
    }
  };

  // apply reduce function
  try {
    ReduceValuesIterator values =
        new ReduceValuesIterator(
            input, reporter, reduceInputValueCounter);

    values.informReduceProgress();
    while (values.more()) {
      reduceInputKeyCounter.increment(1);
      reducer.reduce(values.getKey(), values, collector, reporter);
      values.informReduceProgress();
    }

    // Set progress to 1.0f if there was no exception,
    reporter.setProgress(1.0f);
    
    //Clean up: repeated in catch block below
    reducer.close();
    //End of clean up.
  } catch (IOException ioe) {
    try {
      reducer.close();
    } catch (IOException ignored) {
    }

    throw ioe;
  }
}
Example 6
Source File: ReduceProcessor.java    From tez with Apache License 2.0 4 votes vote down vote up 
void runOldReducer(JobConf job,
    final MRTaskReporter reporter,
    KeyValuesReader input,
    RawComparator comparator,
    Class keyClass,
    Class valueClass,
    final KeyValueWriter output) throws IOException, InterruptedException {

  Reducer reducer =
      ReflectionUtils.newInstance(job.getReducerClass(), job);

  // make output collector

  OutputCollector collector =
      new OutputCollector() {
    public void collect(Object key, Object value)
        throws IOException {
      output.write(key, value);
    }
  };

  // apply reduce function
  try {
    ReduceValuesIterator values =
        new ReduceValuesIterator(
            input, reporter, reduceInputValueCounter);

    values.informReduceProgress();
    while (values.more()) {
      reduceInputKeyCounter.increment(1);
      reducer.reduce(values.getKey(), values, collector, reporter);
      values.informReduceProgress();
    }

    // Set progress to 1.0f if there was no exception,
    reporter.setProgress(1.0f);
    
    //Clean up: repeated in catch block below
    reducer.close();
    //End of clean up.
  } catch (IOException ioe) {
    try {
      reducer.close();
    } catch (IOException ignored) {
    }

    throw ioe;
  }
}