org.apache.hadoop.mapreduce.Reducer.Context Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.Reducer.Context. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Step32.java    From recsys-offline with Apache License 2.0 6 votes vote down vote up
public void map(VarLongWritable key,VectorWritable value,Context context) throws IOException, InterruptedException{  

                long userID=key.get();  
                Vector userVector=value.get();  
                Iterator<Vector.Element> it=userVector.nonZeroes().iterator();  
                IntWritable itemi=new IntWritable();  
                while(it.hasNext()){  
                    Vector.Element e=it.next();  
                    int itemIndex=e.index();  
                    float preferenceValue=(float)e.get();  
                    itemi.set(itemIndex);  
                    context.write(itemi, new VectorOrPrefWritable(userID,preferenceValue));  
                   System.out.println("item :"+itemi+",userand val:"+userID+","+preferenceValue);  
                } 
              
        }
 
Example #2
Source File: MergeReducerBase.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
@Override
public void reduce(Text key, Iterable<MergeRecord> vals, Context c)
    throws IOException, InterruptedException {
  SqoopRecord bestRecord = null;
  try {
    for (MergeRecord val : vals) {
      if (null == bestRecord && !val.isNewRecord()) {
        // Use an old record if we don't have a new record.
        bestRecord = (SqoopRecord) val.getSqoopRecord().clone();
      } else if (val.isNewRecord()) {
        bestRecord = (SqoopRecord) val.getSqoopRecord().clone();
      }
    }
  } catch (CloneNotSupportedException cnse) {
    throw new IOException(cnse);
  }

  if (null != bestRecord) {
    writeRecord(bestRecord, c);
  }
}
 
Example #3
Source File: HistogramRatings.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void map(Object key, Text value, 
    Context context) throws IOException, InterruptedException{

  int rating, reviewIndex, movieIndex;
  String reviews = new String();
  String tok = new String();
  String ratingStr = new String();

  String line = ((Text)value).toString();
  movieIndex = line.indexOf(":");
  if (movieIndex > 0) {
    reviews = line.substring(movieIndex + 1);
    StringTokenizer token = new StringTokenizer(reviews, ",");
    while (token.hasMoreTokens()) {
      tok = token.nextToken();
      reviewIndex = tok.indexOf("_");
      ratingStr = tok.substring(reviewIndex + 1);
      rating = Integer.parseInt(ratingStr);
      context.write(new IntWritable(rating), one);
    }
  }
}
 
Example #4
Source File: RetainMostRecentUpdateConflictResolver.java    From examples with Apache License 2.0 5 votes vote down vote up
/** Returns the most recent document among the colliding updates */
protected Iterator<SolrInputDocument> getMaximum(Iterator<SolrInputDocument> updates, String fieldName,
    Comparator child, Context context) {
  
  SolrInputDocumentComparator comp = new SolrInputDocumentComparator(fieldName, child);
  SolrInputDocument max = null;
  long numDupes = 0;
  long numOutdated = 0;
  while (updates.hasNext()) {
    SolrInputDocument next = updates.next(); 
    assert next != null;
    if (max == null) {
      max = next;
    } else {
      int c = comp.compare(next, max);
      if (c == 0) {
        LOG.debug("Ignoring document version because it is a duplicate: {}", next);
        numDupes++;
      } else if (c > 0) {
        LOG.debug("Ignoring document version because it is outdated: {}", max);
        max = next;
        numOutdated++;
      } else {
        LOG.debug("Ignoring document version because it is outdated: {}", next);        
        numOutdated++;
      }
    }
  }
  
  assert max != null;
  if (numDupes > 0) {
    context.getCounter(COUNTER_GROUP, DUPLICATES_COUNTER_NAME).increment(numDupes);
  }
  if (numOutdated > 0) {
    context.getCounter(COUNTER_GROUP, OUTDATED_COUNTER_NAME).increment(numOutdated);
  }
  return Collections.singletonList(max).iterator();
}
 
Example #5
Source File: Step32.java    From recsys-offline with Apache License 2.0 5 votes vote down vote up
public void reduce(IntWritable key,Iterable<VectorOrPrefWritable> values ,Context context ) throws IOException, InterruptedException{  
      
    for(VectorOrPrefWritable va:values){  
        context.write(key, va);  
        System.err.println("key"+key.toString()+",vlaue"+va);  
    }  
}
 
Example #6
Source File: RejectingUpdateConflictResolver.java    From examples with Apache License 2.0 5 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  SolrInputDocument firstUpdate = null;
  while (updates.hasNext()) {
    if (firstUpdate == null) {
      firstUpdate = updates.next();
      assert firstUpdate != null;
    } else {
      throw new IllegalArgumentException("Update conflict! Documents with the same unique key are forbidden: "
          + key);
    }
  }
  assert firstUpdate != null;
  return Collections.singletonList(firstUpdate).iterator();
}
 
Example #7
Source File: FactDistinctColumnsReducerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void setContextTaskId(final int taskId) {
    Context context = reduceDriver.getContext();
    when(context.getTaskAttemptID()).thenAnswer(new Answer<TaskAttemptID>() {
        @Override
        public TaskAttemptID answer(InvocationOnMock invocation) throws Throwable {
            return TaskAttemptID.forName("attempt__0000_r_" + taskId + "_0");
        }
    });
}
 
Example #8
Source File: MutiWordcount.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void map(Object key, Text value, Context context
                ) throws IOException, InterruptedException {
  StringTokenizer itr = new StringTokenizer(value.toString());
  while (itr.hasMoreTokens()) {
    word.set(itr.nextToken());
    context.write(word, one);
  }
}
 
Example #9
Source File: HistogramRatings.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void reduce(IntWritable key, Iterable<IntWritable> values, Context context
        ) throws IOException, InterruptedException {
  int sum = 0;
  for (IntWritable val : values) {
      sum += val.get();
  }
  context.write(key, new IntWritable(sum));
}
 
Example #10
Source File: MutiWordcount.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void reduce(Text key, Iterable<IntWritable> values, 
                   Context context
                   ) throws IOException, InterruptedException {
  int sum = 0;
  for (IntWritable val : values) {
    sum += val.get();
  }
  result.set(sum);
  context.write(key, result);
}
 
Example #11
Source File: Aggregator.java    From compiler with Apache License 2.0 4 votes vote down vote up
public void setContext(@SuppressWarnings("rawtypes") final Context context) {
	this.context = context;
}
 
Example #12
Source File: MergeReducerBase.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 4 votes vote down vote up
abstract protected void writeRecord(SqoopRecord record, Context c)
throws IOException, InterruptedException;
 
Example #13
Source File: PigMapReduce.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public POPackage getPack(org.apache.hadoop.mapreduce.Reducer.Context context) {
    return ((PigMapReduce.IllustrateReducerContext.IllustratorContext) context).getPack();
}
 
Example #14
Source File: PigMapReduce.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(org.apache.hadoop.mapreduce.Reducer.Context context) {
    return (context instanceof PigMapReduce.IllustrateReducerContext.IllustratorContext);
}
 
Example #15
Source File: PigMapBase.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(Context context) {
    return ((WrappedMapper.Context)context).getConfiguration().get("inIllustrator")!=null;
}
 
Example #16
Source File: PigMapReduce.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public POPackage getPack(
        org.apache.hadoop.mapreduce.Reducer.Context context) {
    return ((PigMapReduce.Reduce.IllustratorContext) context).pack;
}
 
Example #17
Source File: PigMapReduce.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public boolean inIllustrator(
        org.apache.hadoop.mapreduce.Reducer.Context context) {
    return (context instanceof PigMapReduce.Reduce.IllustratorContext);
}
 
Example #18
Source File: SortingUpdateConflictResolver.java    From examples with Apache License 2.0 4 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return sort(updates, getOrderByFieldName(), new SolrInputDocumentComparator.TimeStampComparator());
}
 
Example #19
Source File: NoChangeUpdateConflictResolver.java    From examples with Apache License 2.0 4 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return updates;
}
 
Example #20
Source File: RetainMostRecentUpdateConflictResolver.java    From examples with Apache License 2.0 4 votes vote down vote up
@Override
public Iterator<SolrInputDocument> orderUpdates(Text key, Iterator<SolrInputDocument> updates, Context ctx) {    
  return getMaximum(updates, getOrderByFieldName(), new SolrInputDocumentComparator.TimeStampComparator(), ctx);
}
 
Example #21
Source File: Aggregator.java    From compiler with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
public Context getContext() {
	return this.context;
}
 
Example #22
Source File: PigMapBase.java    From spork with Apache License 2.0 3 votes vote down vote up
/**
 * 
 * Get mapper's illustrator context
 * 
 * @param conf  Configuration
 * @param input Input bag to serve as data source
 * @param output Map output buffer
 * @param split the split
 * @return Illustrator's context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Configuration conf, DataBag input,
      List<Pair<PigNullableWritable, Writable>> output, InputSplit split)
      throws IOException, InterruptedException {
	org.apache.hadoop.mapreduce.Mapper.Context mapperContext = new WrappedMapper<Text, Tuple, PigNullableWritable, Writable>().getMapContext(new IllustratorContext(conf, input, output, split));
    return mapperContext;
}
 
Example #23
Source File: PigMapReduce.java    From spork with Apache License 2.0 3 votes vote down vote up
/**
 * Get reducer's illustrator context
 * 
 * @param input Input buffer as output by maps
 * @param pkg package
 * @return reducer's illustrator context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Job job,
       List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
	org.apache.hadoop.mapreduce.Reducer.Context reducerContext = new IllustrateReducerContext()
			.getReducerContext(new IllustratorContextImpl(job, input, pkg));
	return reducerContext;
}
 
Example #24
Source File: PigMapReduce.java    From spork with Apache License 2.0 2 votes vote down vote up
/**
 * Get reducer's illustrator context
 * 
 * @param input Input buffer as output by maps
 * @param pkg package
 * @return reducer's illustrator context
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public Context getIllustratorContext(Job job,
       List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
    return new IllustratorContext(job, input, pkg);
}
 
Example #25
Source File: UpdateConflictResolver.java    From examples with Apache License 2.0 2 votes vote down vote up
/**
 * Given a list of all colliding document updates for the same unique document
 * key, this method returns zero or more documents in an application specific
 * order.
 * 
 * The caller will then apply the updates for this key to Solr in the order
 * returned by the orderUpdate() method.
 * 
 * @param uniqueKey
 *          the document key common to all collidingUpdates mentioned below
 * @param collidingUpdates
 *          all updates in the MapReduce job that have a key equal to
 *          {@code uniqueKey} mentioned above. The input order is unspecified.
 * @param context
 *          The <code>Context</code> passed from the {@link Reducer}
 *          implementations.
 * @return the order in which the updates shall be applied to Solr
 */
Iterator<SolrInputDocument> orderUpdates(
    Text uniqueKey, Iterator<SolrInputDocument> collidingUpdates, Context context);