Java Code Examples for org.apache.hadoop.mapreduce.Reducer.Context

The following examples show how to use org.apache.hadoop.mapreduce.Reducer.Context. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void reduce(Text key, Iterable<MergeRecord> vals, Context c)
    throws IOException, InterruptedException {
  SqoopRecord bestRecord = null;
  try {
    for (MergeRecord val : vals) {
      if (null == bestRecord && !val.isNewRecord()) {
        // Use an old record if we don't have a new record.
        bestRecord = (SqoopRecord) val.getSqoopRecord().clone();
      } else if (val.isNewRecord()) {
        bestRecord = (SqoopRecord) val.getSqoopRecord().clone();
      }
    }
  } catch (CloneNotSupportedException cnse) {
    throw new IOException(cnse);
  }

  if (null != bestRecord) {
    writeRecord(bestRecord, c);
  }
}
 
Example 2
Source Project: big-c   Source File: HistogramRatings.java    License: Apache License 2.0 6 votes vote down vote up
public void map(Object key, Text value, 
    Context context) throws IOException, InterruptedException{

  int rating, reviewIndex, movieIndex;
  String reviews = new String();
  String tok = new String();
  String ratingStr = new String();

  String line = ((Text)value).toString();
  movieIndex = line.indexOf(":");
  if (movieIndex > 0) {
    reviews = line.substring(movieIndex + 1);
    StringTokenizer token = new StringTokenizer(reviews, ",");
    while (token.hasMoreTokens()) {
      tok = token.nextToken();
      reviewIndex = tok.indexOf("_");
      ratingStr = tok.substring(reviewIndex + 1);
      rating = Integer.parseInt(ratingStr);
      context.write(new IntWritable(rating), one);
    }
  }
}
 
Example 3
Source Project: recsys-offline   Source File: Step32.java    License: Apache License 2.0 6 votes vote down vote up
public void map(VarLongWritable key,VectorWritable value,Context context) throws IOException, InterruptedException{  

                long userID=key.get();  
                Vector userVector=value.get();  
                Iterator<Vector.Element> it=userVector.nonZeroes().iterator();  
                IntWritable itemi=new IntWritable();  
                while(it.hasNext()){  
                    Vector.Element e=it.next();  
                    int itemIndex=e.index();  
                    float preferenceValue=(float)e.get();  
                    itemi.set(itemIndex);  
                    context.write(itemi, new VectorOrPrefWritable(userID,preferenceValue));  
                   System.out.println("item :"+itemi+",userand val:"+userID+","+preferenceValue);  
                } 
              
        }
 
Example 4
Source Project: big-c   Source File: MutiWordcount.java    License: Apache License 2.0 5 votes vote down vote up
public void map(Object key, Text value, Context context
                ) throws IOException, InterruptedException {
  StringTokenizer itr = new StringTokenizer(value.toString());
  while (itr.hasMoreTokens()) {
    word.set(itr.nextToken());
    context.write(word, one);
  }
}
 
Example 5
Source Project: big-c   Source File: MutiWordcount.java    License: Apache License 2.0 5 votes vote down vote up
public void reduce(Text key, Iterable<IntWritable> values, 
                   Context context
                   ) throws IOException, InterruptedException {
  int sum = 0;
  for (IntWritable val : values) {
    sum += val.get();
  }
  result.set(sum);
  context.write(key, result);
}
 
Example 6
Source Project: big-c   Source File: HistogramRatings.java    License: Apache License 2.0 5 votes vote down vote up
public void reduce(IntWritable key, Iterable<IntWritable> values, Context context
        ) throws IOException, InterruptedException {
  int sum = 0;
  for (IntWritable val : values) {
      sum += val.get();
  }
  context.write(key, new IntWritable(sum));
}
 
Example 7
Source Project: kylin   Source File: FactDistinctColumnsReducerTest.java    License: Apache License 2.0 5 votes vote down vote up
private void setContextTaskId(final int taskId) {
    Context context = reduceDriver.getContext();
    when(context.getTaskAttemptID()).thenAnswer(new Answer<TaskAttemptID>() {
        @Override
        public TaskAttemptID answer(InvocationOnMock invocation) throws Throwable {
            return TaskAttemptID.forName("attempt__0000_r_" + taskId + "_0");
        }
    });
}
 
Example 8
Source Project: examples   Source File: RejectingUpdateConflictResolver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  SolrInputDocument firstUpdate = null;
  while (updates.hasNext()) {
    if (firstUpdate == null) {
      firstUpdate = updates.next();
      assert firstUpdate != null;
    } else {
      throw new IllegalArgumentException("Update conflict! Documents with the same unique key are forbidden: "
          + key);
    }
  }
  assert firstUpdate != null;
  return Collections.singletonList(firstUpdate).iterator();
}
 
Example 9
/** Returns the most recent document among the colliding updates */
protected Iterator<SolrInputDocument> getMaximum(Iterator<SolrInputDocument> updates, String fieldName,
    Comparator child, Context context) {
  
  SolrInputDocumentComparator comp = new SolrInputDocumentComparator(fieldName, child);
  SolrInputDocument max = null;
  long numDupes = 0;
  long numOutdated = 0;
  while (updates.hasNext()) {
    SolrInputDocument next = updates.next(); 
    assert next != null;
    if (max == null) {
      max = next;
    } else {
      int c = comp.compare(next, max);
      if (c == 0) {
        LOG.debug("Ignoring document version because it is a duplicate: {}", next);
        numDupes++;
      } else if (c > 0) {
        LOG.debug("Ignoring document version because it is outdated: {}", max);
        max = next;
        numOutdated++;
      } else {
        LOG.debug("Ignoring document version because it is outdated: {}", next);        
        numOutdated++;
      }
    }
  }
  
  assert max != null;
  if (numDupes > 0) {
    context.getCounter(COUNTER_GROUP, DUPLICATES_COUNTER_NAME).increment(numDupes);
  }
  if (numOutdated > 0) {
    context.getCounter(COUNTER_GROUP, OUTDATED_COUNTER_NAME).increment(numOutdated);
  }
  return Collections.singletonList(max).iterator();
}
 
Example 10
Source Project: recsys-offline   Source File: Step32.java    License: Apache License 2.0 5 votes vote down vote up
public void reduce(IntWritable key,Iterable<VectorOrPrefWritable> values ,Context context ) throws IOException, InterruptedException{  
      
    for(VectorOrPrefWritable va:values){  
        context.write(key, va);  
        System.err.println("key"+key.toString()+",vlaue"+va);  
    }  
}
 
Example 11
abstract protected void writeRecord(SqoopRecord record, Context c)
throws IOException, InterruptedException;
 
Example 12
Source Project: compiler   Source File: Aggregator.java    License: Apache License 2.0 4 votes vote down vote up
public void setContext(@SuppressWarnings("rawtypes") final Context context) {
	this.context = context;
}
 
Example 13
Source Project: compiler   Source File: Aggregator.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
public Context getContext() {
	return this.context;
}
 
Example 14
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return getMaximum(updates, getOrderByFieldName(), new SolrInputDocumentComparator.TimeStampComparator(), ctx);
}
 
Example 15
Source Project: examples   Source File: NoChangeUpdateConflictResolver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return updates;
}
 
Example 16
Source Project: examples   Source File: SortingUpdateConflictResolver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return sort(updates, getOrderByFieldName(), new SolrInputDocumentComparator.TimeStampComparator());
}
 
Example 17
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(
        org.apache.hadoop.mapreduce.Reducer.Context context) {
    return (context instanceof PigMapReduce.Reduce.IllustratorContext);
}
 
Example 18
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public POPackage getPack(
        org.apache.hadoop.mapreduce.Reducer.Context context) {
    return ((PigMapReduce.Reduce.IllustratorContext) context).pack;
}
 
Example 19
Source Project: spork   Source File: PigMapBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(Context context) {
    return ((WrappedMapper.Context)context).getConfiguration().get("inIllustrator")!=null;
}
 
Example 20
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(org.apache.hadoop.mapreduce.Reducer.Context context) {
    return (context instanceof PigMapReduce.IllustrateReducerContext.IllustratorContext);
}
 
Example 21
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public POPackage getPack(org.apache.hadoop.mapreduce.Reducer.Context context) {
    return ((PigMapReduce.IllustrateReducerContext.IllustratorContext) context).getPack();
}
 
Example 22
Source Project: spork   Source File: PigMapBase.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * 
 * Get mapper's illustrator context
 * 
 * @param conf  Configuration
 * @param input Input bag to serve as data source
 * @param output Map output buffer
 * @param split the split
 * @return Illustrator's context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Configuration conf, DataBag input,
      List<Pair<PigNullableWritable, Writable>> output, InputSplit split)
      throws IOException, InterruptedException {
	org.apache.hadoop.mapreduce.Mapper.Context mapperContext = new WrappedMapper<Text, Tuple, PigNullableWritable, Writable>().getMapContext(new IllustratorContext(conf, input, output, split));
    return mapperContext;
}
 
Example 23
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Get reducer's illustrator context
 * 
 * @param input Input buffer as output by maps
 * @param pkg package
 * @return reducer's illustrator context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Job job,
       List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
	org.apache.hadoop.mapreduce.Reducer.Context reducerContext = new IllustrateReducerContext()
			.getReducerContext(new IllustratorContextImpl(job, input, pkg));
	return reducerContext;
}
 
Example 24
Source Project: examples   Source File: UpdateConflictResolver.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Given a list of all colliding document updates for the same unique document
 * key, this method returns zero or more documents in an application specific
 * order.
 * 
 * The caller will then apply the updates for this key to Solr in the order
 * returned by the orderUpdate() method.
 * 
 * @param uniqueKey
 *          the document key common to all collidingUpdates mentioned below
 * @param collidingUpdates
 *          all updates in the MapReduce job that have a key equal to
 *          {@code uniqueKey} mentioned above. The input order is unspecified.
 * @param context
 *          The <code>Context</code> passed from the {@link Reducer}
 *          implementations.
 * @return the order in which the updates shall be applied to Solr
 */
Iterator<SolrInputDocument> orderUpdates(
    Text uniqueKey, Iterator<SolrInputDocument> collidingUpdates, Context context);
 
Example 25
Source Project: spork   Source File: PigMapReduce.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get reducer's illustrator context
 * 
 * @param input Input buffer as output by maps
 * @param pkg package
 * @return reducer's illustrator context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Job job,
       List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
    return new IllustratorContext(job, input, pkg);
}