org.apache.hadoop.mapreduce.TaskInputOutputContext Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.TaskInputOutputContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DatasetSourceTarget.java    From kite with Apache License 2.0 6 votes vote down vote up
@Override
public ReadableData<E> asReadable() {
  return new ReadableData<E>() {

    @Override
    public Set<SourceTarget<?>> getSourceTargets() {
      return ImmutableSet.<SourceTarget<?>>of(DatasetSourceTarget.this);
    }

    @Override
    public void configure(Configuration conf) {
      // TODO: optimize for file-based datasets by using distributed cache
    }

    @Override
    public Iterable<E> read(TaskInputOutputContext<?, ?, ?, ?> context) throws IOException {
      return view.newReader();
    }
  };
}
 
Example #2
Source File: MultiTableRangePartitioner.java    From datawave with Apache License 2.0 6 votes vote down vote up
@Override
public int getPartition(BulkIngestKey key, Value value, int numPartitions) {
    readCacheFilesIfNecessary();
    
    String tableName = key.getTableName().toString();
    Text[] cutPointArray = splitsByTable.get().get(tableName);
    
    if (null == cutPointArray)
        return (tableName.hashCode() & Integer.MAX_VALUE) % numPartitions;
    key.getKey().getRow(holder);
    int index = Arrays.binarySearch(cutPointArray, holder);
    index = calculateIndex(index, numPartitions, tableName, cutPointArray.length);
    
    index = partitionLimiter.limit(numPartitions, index);
    
    TaskInputOutputContext<?,?,?,?> c = context;
    if (c != null && collectStats) {
        c.getCounter("Partitions: " + key.getTableName(), "part." + formatter.format(index)).increment(1);
    }
    
    return index;
}
 
Example #3
Source File: Chain.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Add mapper that reads and writes from/to the queue
 */
@SuppressWarnings("unchecked")
void addMapper(ChainBlockingQueue<KeyValuePair<?, ?>> input,
    ChainBlockingQueue<KeyValuePair<?, ?>> output,
    TaskInputOutputContext context, int index) throws IOException,
    InterruptedException {
  Configuration conf = getConf(index);
  Class<?> keyClass = conf.getClass(MAPPER_INPUT_KEY_CLASS, Object.class);
  Class<?> valueClass = conf.getClass(MAPPER_INPUT_VALUE_CLASS, Object.class);
  Class<?> keyOutClass = conf.getClass(MAPPER_OUTPUT_KEY_CLASS, Object.class);
  Class<?> valueOutClass = conf.getClass(MAPPER_OUTPUT_VALUE_CLASS,
      Object.class);
  RecordReader rr = new ChainRecordReader(keyClass, valueClass, input, conf);
  RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass, output,
      conf);
  MapRunner runner = new MapRunner(mappers.get(index), createMapContext(rr,
      rw, context, getConf(index)), rr, rw);
  threads.add(runner);
}
 
Example #4
Source File: HalvadeFileUtils.java    From halvade with GNU General Public License v3.0 6 votes vote down vote up
protected static int attemptDownloadFileFromHDFS(TaskInputOutputContext context, FileSystem fs, String from, String to, int tries) throws IOException {
    if(from.equalsIgnoreCase(to)) return 0;
    int val = privateDownloadFileFromHDFS(context, fs, from, to);
    int try_ = 1;
    while (val != 0 && try_ < tries) {
        val = privateDownloadFileFromHDFS(context, fs, from, to);
        try_++;
    }
    if(val == 0)
        Logger.DEBUG(from + " downloaded");
    else {
        Logger.DEBUG(from + " failed to download");   
        throw new IOException("expection downloading "  + from + " to " + to + " from fs <" + fs + "> with error val: " + val);
    }
    return val;            
}
 
Example #5
Source File: TableCachingContextWriter.java    From datawave with Apache License 2.0 6 votes vote down vote up
@Override
protected void flush(Multimap<BulkIngestKey,Value> entries, TaskInputOutputContext<?,?,BulkIngestKey,Value> context) throws IOException,
                InterruptedException {
    Multimap<BulkIngestKey,Value> residual = HashMultimap.create();
    for (BulkIngestKey key : entries.keySet()) {
        Collection<Value> values = entries.get(key);
        if (tableCacheConf.containsKey(key.getTableName())) {
            cache(key, values, context);
        } else {
            residual.putAll(key, values);
        }
    }
    if (!residual.isEmpty()) {
        contextWriter.write(residual, context);
    }
}
 
Example #6
Source File: Chain.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Add mapper(the first mapper) that reads input from the input
 * context and writes to queue
 */
@SuppressWarnings("unchecked")
void addMapper(TaskInputOutputContext inputContext,
    ChainBlockingQueue<KeyValuePair<?, ?>> output, int index)
    throws IOException, InterruptedException {
  Configuration conf = getConf(index);
  Class<?> keyOutClass = conf.getClass(MAPPER_OUTPUT_KEY_CLASS, Object.class);
  Class<?> valueOutClass = conf.getClass(MAPPER_OUTPUT_VALUE_CLASS,
      Object.class);

  RecordReader rr = new ChainRecordReader(inputContext);
  RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass, output,
      conf);
  Mapper.Context mapperContext = createMapContext(rr, rw,
      (MapContext) inputContext, getConf(index));
  MapRunner runner = new MapRunner(mappers.get(index), mapperContext, rr, rw);
  threads.add(runner);
}
 
Example #7
Source File: Chain.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Add reducer that reads from context and writes to a queue
 */
@SuppressWarnings("unchecked")
void addReducer(TaskInputOutputContext inputContext,
    ChainBlockingQueue<KeyValuePair<?, ?>> outputQueue) throws IOException,
    InterruptedException {

  Class<?> keyOutClass = rConf.getClass(REDUCER_OUTPUT_KEY_CLASS,
      Object.class);
  Class<?> valueOutClass = rConf.getClass(REDUCER_OUTPUT_VALUE_CLASS,
      Object.class);
  RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass,
      outputQueue, rConf);
  Reducer.Context reducerContext = createReduceContext(rw,
      (ReduceContext) inputContext, rConf);
  ReduceRunner runner = new ReduceRunner(reducerContext, reducer, rw);
  threads.add(runner);
}
 
Example #8
Source File: HadoopMultipleOutputFormat.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    super.open(taskNumber, numTasks);

    synchronized (OPEN_MULTIPLE_MUTEX) {
        try {
            TaskInputOutputContext taskInputOutputContext = new ReduceContextImpl(configuration,
                    context.getTaskAttemptID(), new InputIterator(), new GenericCounter(), new GenericCounter(),
                    recordWriter, outputCommitter, new DummyReporter(), null,
                    BytesWritable.class, BytesWritable.class);
            this.writer = new MultipleOutputs(taskInputOutputContext);
        } catch (InterruptedException e) {
            throw new IOException("Could not create MultipleOutputs.", e);
        }
    }
}
 
Example #9
Source File: AbstractContextWriter.java    From datawave with Apache License 2.0 6 votes vote down vote up
/**
 * Write the key, value to the cache.
 */
@Override
public void write(BulkIngestKey key, Value value, TaskInputOutputContext<?,?,OK,OV> context) throws IOException, InterruptedException {
    if (constraintChecker != null && constraintChecker.isConfigured()) {
        constraintChecker.check(key.getTableName(), key.getKey().getColumnVisibilityData().getBackingArray());
    }
    
    cache.put(key, value);
    this.count++;
    if (counters != null) {
        counters.incrementCounter(key);
    }
    if (cache.size() > this.maxSize) {
        commit(context);
    }
}
 
Example #10
Source File: ProtobufEdgeDataTypeHandler.java    From datawave with Apache License 2.0 6 votes vote down vote up
protected long writeEdges(EdgeDataBundle value, TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context,
                ContextWriter<KEYOUT,VALUEOUT> contextWriter, boolean validActivtyDate, boolean sameActivityDate, long eventDate) throws IOException,
                InterruptedException {
    
    long edgesCreated = 0;
    if (eventDate < newFormatStartDate) {
        edgesCreated += writeEdges(value, context, contextWriter, EdgeKey.DATE_TYPE.OLD_EVENT);
    } else {
        if (validActivtyDate) {
            if (sameActivityDate) {
                edgesCreated += writeEdges(value, context, contextWriter, EdgeKey.DATE_TYPE.ACTIVITY_AND_EVENT);
            } else {
                edgesCreated += writeEdges(value, context, contextWriter, EdgeKey.DATE_TYPE.ACTIVITY_ONLY);
                edgesCreated += writeEdges(value, context, contextWriter, EdgeKey.DATE_TYPE.EVENT_ONLY);
            }
        } else {
            edgesCreated += writeEdges(value, context, contextWriter, EdgeKey.DATE_TYPE.EVENT_ONLY);
        }
    }
    
    return edgesCreated;
}
 
Example #11
Source File: Chain.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Add reducer that reads from context and writes to a queue
 */
@SuppressWarnings("unchecked")
void addReducer(TaskInputOutputContext inputContext,
    ChainBlockingQueue<KeyValuePair<?, ?>> outputQueue) throws IOException,
    InterruptedException {

  Class<?> keyOutClass = rConf.getClass(REDUCER_OUTPUT_KEY_CLASS,
      Object.class);
  Class<?> valueOutClass = rConf.getClass(REDUCER_OUTPUT_VALUE_CLASS,
      Object.class);
  RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass,
      outputQueue, rConf);
  Reducer.Context reducerContext = createReduceContext(rw,
      (ReduceContext) inputContext, rConf);
  ReduceRunner runner = new ReduceRunner(reducerContext, reducer, rw);
  threads.add(runner);
}
 
Example #12
Source File: DistSum.java    From big-c with Apache License 2.0 6 votes vote down vote up
/** Compute sigma */
static void compute(Summation sigma,
    TaskInputOutputContext<?, ?, NullWritable, TaskResult> context
    ) throws IOException, InterruptedException {
  String s;
  LOG.info(s = "sigma=" + sigma);
  context.setStatus(s);

  final long start = System.currentTimeMillis();
  sigma.compute();
  final long duration = System.currentTimeMillis() - start;
  final TaskResult result = new TaskResult(sigma, duration);

  LOG.info(s = "result=" + result);
  context.setStatus(s);
  context.write(NullWritable.get(), result);
}
 
Example #13
Source File: CollapsingMapper.java    From datafu with Apache License 2.0 5 votes vote down vote up
@Override
public void setContext(TaskInputOutputContext<Object,Object,Object,Object> context)
{      
  super.setContext(context);
  
  if (_mapper instanceof Configurable)
  {
    ((Configurable)_mapper).setConf(context.getConfiguration());
  }
}
 
Example #14
Source File: ExtendedContentIndexingColumnBasedHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
private void completePhrase(StringBuilder baseTerm, List<Collection<String>> terms, String zone, RawRecordContainer event, int position,
                BloomFilter alreadyIndexedTerms, TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context,
                ContextWriter<KEYOUT,VALUEOUT> contextWriter, StatusReporter reporter) throws IOException, InterruptedException {
    if (terms.isEmpty()) {
        return;
    }
    for (String term : terms.get(0)) {
        if (term == null) {
            continue;
        }
        boolean properLen = term.length() >= tokenHelper.getTermLengthMinimum();
        // Add the current term and emit the phrase if the current term isn't empty
        if (properLen) {
            baseTerm.append(SPACE).append(term);
            
            counters.increment(ContentIndexCounters.PHRASES_PROCESSED_COUNTER, reporter);
            
            processTermAndZone(event, position, new TermAndZone(baseTerm.toString(), zone), alreadyIndexedTerms, context, contextWriter, reporter);
        }
        
        // If we have more terms to add to this phrase, recurse
        if (terms.size() > 1) {
            completePhrase(baseTerm, terms.subList(1, terms.size()), zone, event, position, alreadyIndexedTerms, context, contextWriter, reporter);
        }
        
        // Only remove the space and term if we actually added one
        if (properLen) {
            // Remove the space and the token we appended last
            baseTerm.setLength(baseTerm.length() - 1 - term.length());
        }
    }
}
 
Example #15
Source File: PartitioningReducer.java    From datafu with Apache License 2.0 5 votes vote down vote up
@Override
public void setContext(TaskInputOutputContext<Object,Object,Object,Object> context)
{           
  super.setContext(context);
  
  // ... and we also write the final output to multiple directories
  _multipleOutputs = new AvroMultipleOutputs(context);
}
 
Example #16
Source File: WikipediaDataTypeHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * Tokenize the event, and write all of the shard, shardIndex, and shardReverseIndex keys out to the context
 * 
 * @param event
 * @param context
 * @param contextWriter
 * @return
 * @throws IOException
 * @throws InterruptedException
 */
@Override
protected long tokenizeEvent(RawRecordContainer event, TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context,
                ContextWriter<KEYOUT,VALUEOUT> contextWriter, StatusReporter reporter) throws IOException, InterruptedException {
    
    long count = 0;
    
    final byte[] visibility = flatten(event.getVisibility());
    final byte[] rawData = event.getRawData();
    
    Document root;
    try {
        root = this.parser.parse(new ByteArrayInputStream(rawData));
    } catch (SAXException e) {
        throw new RuntimeException(e);
    }
    
    NodeList revisions = root.getElementsByTagName("revision");
    
    // For each revision, try to find the stuff we want to tokenize
    for (int i = 0; i < revisions.getLength(); i++) {
        Node revision = revisions.item(i);
        NodeList children = revision.getChildNodes();
        for (int j = 0; j < children.getLength(); j++) {
            Node revChild = children.item(j);
            
            if (REVISION_COMMENT.equals(revChild.getNodeName())) {
                count += tokenizeTextNode(revChild.getTextContent(), event, visibility, context, contextWriter, REVISION_COMMENT_FIELD_NAME,
                                REVISION_COMMENT_TOKEN, reporter);
            } else if (REVISION_TEXT.equals(revChild.getNodeName())) {
                count += tokenizeTextNode(revChild.getTextContent(), event, visibility, context, contextWriter, REVISION_TEXT_FIELD_NAME,
                                REVISION_TEXT_TOKEN, reporter);
            }
        }
    }
    
    return count;
}
 
Example #17
Source File: ExtendedContentIndexingColumnBasedHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Override
public long process(KEYIN key, RawRecordContainer event, Multimap<String,NormalizedContentInterface> eventFields,
                TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context, ContextWriter<KEYOUT,VALUEOUT> contextWriter)
                throws IOException, InterruptedException {
    
    // Hold some event-specific variables to avoid re-processing
    this.shardId = getShardId(event);
    
    if (tokenHelper.isVerboseShardCounters()) {
        context.getCounter("EVENT_SHARD_ID", new String(this.shardId)).increment(1);
    }
    
    this.eventDataTypeName = event.getDataType().outputName();
    this.eventUid = event.getId().toString();
    
    // write the standard set of keys
    Multimap<BulkIngestKey,Value> keys = super.processBulk(key, event, eventFields, new ContextWrappedStatusReporter(context));
    long count = keys.size();
    contextWriter.write(keys, context);
    
    StatusReporter reporter = new ContextWrappedStatusReporter(context);
    
    // gc before we get into the tokenization piece
    keys = null;
    
    // stream the tokens to the context writer here
    count += tokenizeEvent(event, context, contextWriter, reporter);
    
    // return the number of records written
    return count;
}
 
Example #18
Source File: WikipediaDataTypeHandlerTest.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Override
protected void flush(Multimap<BulkIngestKey,Value> entries, TaskInputOutputContext<?,?,BulkIngestKey,Value> context) throws IOException,
                InterruptedException {
    for (Map.Entry<BulkIngestKey,Value> entry : entries.entries()) {
        cache.put(entry.getKey(), entry.getValue());
    }
}
 
Example #19
Source File: ExtendedContentIndexingColumnBasedHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and writes the BulkIngestKey for the event's field and global indexes to the ContextWriter
 * 
 * @param event
 * @param contextWriter
 * @param context
 * @param nFV
 * @param shardId
 * @param fieldVisibility
 * @throws IOException
 * @throws InterruptedException
 */
protected void createShardIndexColumns(RawRecordContainer event, ContextWriter<KEYOUT,VALUEOUT> contextWriter,
                TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context, NormalizedContentInterface nFV, byte[] shardId,
                byte[] fieldVisibility) throws IOException, InterruptedException {
    
    if (log.isDebugEnabled()) {
        log.debug("Creating a mutation for " + nFV.getIndexedFieldValue() + ':' + nFV.getIndexedFieldName());
    }
    
    // Still need the field index record for the token
    createShardFieldIndexColumn(event, contextWriter, context, nFV, shardId, null, fieldVisibility, this.ingestHelper.getReplaceMalformedUTF8(),
                    this.ingestHelper.getDeleteMode());
    
    // If we're creating index terms
    if ((null != this.getShardIndexTableName()) && this.ingestHelper != null) {
        if (this.ingestHelper.isIndexedField(nFV.getIndexedFieldName())) {
            // Throw it into the index
            createIndexColumn(event, contextWriter, context, nFV, shardId, this.getShardIndexTableName(), fieldVisibility,
                            this.ingestHelper.getReplaceMalformedUTF8(), this.ingestHelper.getDeleteMode());
        }
    }
    
    // If we're creating reverse index terms
    if ((null != this.getShardReverseIndexTableName()) && this.ingestHelper != null) {
        if (this.ingestHelper.isReverseIndexedField(nFV.getIndexedFieldName())) {
            // Throw the reversed term into the reverse index
            NormalizedContentInterface reverseNfv = new NormalizedFieldAndValue(nFV);
            reverseNfv.setIndexedFieldValue(new StringBuilder(nFV.getIndexedFieldValue()).reverse().toString());
            createIndexColumn(event, contextWriter, context, reverseNfv, shardId, this.getShardReverseIndexTableName(), fieldVisibility,
                            this.ingestHelper.getReplaceMalformedUTF8(), this.ingestHelper.getDeleteMode());
        }
    }
}
 
Example #20
Source File: WikipediaDataTypeHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and writes the BulkIngestKey for the event's field/value to the ContextWriter (instead of the Multimap that the {@link ShardedDataTypeHandler}
 * uses).
 * 
 * @param event
 * @param contextWriter
 * @param context
 * @param nFV
 * @param shardId
 * @param visibility
 * @throws IOException
 * @throws InterruptedException
 */
@Override
protected void createShardEventColumn(RawRecordContainer event, ContextWriter<KEYOUT,VALUEOUT> contextWriter,
                TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context, NormalizedContentInterface nFV, byte[] shardId,
                byte[] visibility) throws IOException, InterruptedException {
    
    String fieldName = nFV.getEventFieldName();
    String fieldValue = nFV.getEventFieldValue();
    
    if (this.ingestHelper.isIndexOnlyField(fieldName))
        return;
    
    if (this.ingestHelper.isCompositeField(fieldName) && !this.ingestHelper.isOverloadedCompositeField(fieldName))
        return;
    
    if (StringUtils.isEmpty(fieldValue))
        return;
    
    Text colf = new Text(event.getDataType().outputName());
    TextUtil.textAppend(colf, event.getId().toString(), this.eventReplaceMalformedUTF8);
    
    Text colq = new Text(fieldName);
    TextUtil.textAppend(colq, fieldValue, this.ingestHelper.getReplaceMalformedUTF8());
    Key k = createKey(shardId, colf, colq, visibility, event.getDate(), this.ingestHelper.getDeleteMode());
    BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), k);
    contextWriter.write(bKey, DataTypeHandler.NULL_VALUE, context);
}
 
Example #21
Source File: ProtobufEdgeDataTypeHandler.java    From datawave with Apache License 2.0 5 votes vote down vote up
protected int writeKey(Key key, Value val, TaskInputOutputContext<KEYIN,? extends RawRecordContainer,KEYOUT,VALUEOUT> context,
                ContextWriter<KEYOUT,VALUEOUT> contextWriter) throws IOException, InterruptedException {
    if (key == null)
        return 0;
    BulkIngestKey bk = new BulkIngestKey(new Text(this.edgeTableName), key);
    contextWriter.write(bk, val, context);
    return 1;
}
 
Example #22
Source File: AggregatingReducer.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * This method is called once for each key. Most applications will define their reduce class by overriding this method. The default implementation is an
 * identity function.
 */
@SuppressWarnings("unchecked")
public void doReduce(IK key, Iterable<IV> values, TaskInputOutputContext<?,?,OK,OV> context) throws IOException, InterruptedException {
    for (IV value : values) {
        context.write((OK) key, (OV) value);
    }
}
 
Example #23
Source File: NewAPIHadoopCounterReporterTest.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
@BeforeClass
@SuppressWarnings("unchecked")
public void setUp() {
  TaskInputOutputContext<Object, Object, Object, Object> mockContext = Mockito.mock(TaskInputOutputContext.class);

  this.recordsProcessedCount = Mockito.mock(Counter.class);
  Mockito.when(mockContext.getCounter(
      this.name, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName())))
      .thenReturn(this.recordsProcessedCount);

  this.recordProcessRateCount = Mockito.mock(Counter.class);
  Mockito.when(mockContext.getCounter(
      this.name, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName())))
      .thenReturn(this.recordProcessRateCount);

  this.recordSizeDistributionCount = Mockito.mock(Counter.class);
  Mockito.when(mockContext.getCounter(
      this.name, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName())))
      .thenReturn(this.recordSizeDistributionCount);

  this.totalDurationCount = Mockito.mock(Counter.class);
  Mockito.when(mockContext.getCounter(
      this.name, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName())))
      .thenReturn(this.totalDurationCount);

  this.queueSize = Mockito.mock(Counter.class);
  Mockito.when(mockContext.getCounter(this.name, QUEUE_SIZE)).thenReturn(this.queueSize);

  this.hadoopCounterReporter = NewAPIHadoopCounterReporter.builder(mockContext)
      .convertRatesTo(TimeUnit.SECONDS)
      .convertDurationsTo(TimeUnit.SECONDS)
      .filter(MetricFilter.ALL)
      .build(MetricContext.builder(this.name).build());
}
 
Example #24
Source File: MetricsService.java    From datawave with Apache License 2.0 5 votes vote down vote up
public MetricsService(ContextWriter<OK,OV> contextWriter, TaskInputOutputContext<?,?,OK,OV> context) {
    Configuration conf = context.getConfiguration();
    
    this.date = DateHelper.format(new Date());
    
    this.fieldNames = MetricsConfiguration.getFieldNames(conf);
    this.enabledLabels = MetricsConfiguration.getLabels(conf);
    this.enabledKeys = enabledLabels.keySet();
    
    this.wildcardedLabels = new HashSet<>();
    for (Map.Entry<String,String> entry : enabledLabels.entries()) {
        if (WILDCARD.equals(entry.getValue())) {
            wildcardedLabels.add(entry.getKey());
        }
    }
    
    this.receivers = new HashMap<>();
    for (MetricsReceiver receiver : MetricsConfiguration.getReceivers(conf)) {
        this.receivers.put(receiver.getMetric(), receiver);
        receiver.configure(conf, date);
    }
    
    this.store = new AggregatingMetricsStore<>(contextWriter, context);
    
    if (logger.isInfoEnabled()) {
        logger.info("Metrics Service Initialized");
        logger.info("enabledLabels = " + enabledLabels);
        logger.info("receivers = " + receivers);
        logger.info("fieldNames = " + fieldNames);
    }
    
    Preconditions.checkNotNull(fieldNames);
    Preconditions.checkArgument(!enabledLabels.isEmpty());
    Preconditions.checkArgument(!receivers.isEmpty());
}
 
Example #25
Source File: ProtobufEdgeDeleteModeTest.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Override
protected void flush(Multimap<BulkIngestKey,Value> entries, TaskInputOutputContext<?,?,BulkIngestKey,Value> context) throws IOException,
                InterruptedException {
    for (Map.Entry<BulkIngestKey,Value> entry : entries.entries()) {
        cache.put(entry.getKey(), entry.getValue());
    }
}
 
Example #26
Source File: Chain.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
<KEYIN, VALUEIN, KEYOUT, VALUEOUT> void runReducer(
    TaskInputOutputContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context)
    throws IOException, InterruptedException {
  RecordWriter<KEYOUT, VALUEOUT> rw = new ChainRecordWriter<KEYOUT, VALUEOUT>(
      context);
  Reducer.Context reducerContext = createReduceContext(rw,
      (ReduceContext) context, rConf);
  reducer.run(reducerContext);
  rw.close(context);
}
 
Example #27
Source File: Chain.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Create a map context that is based on ChainMapContext and the given record
 * reader and record writer
 */
private <KEYIN, VALUEIN, KEYOUT, VALUEOUT> 
Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context createMapContext(
    RecordReader<KEYIN, VALUEIN> rr, RecordWriter<KEYOUT, VALUEOUT> rw,
    TaskInputOutputContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context,
    Configuration conf) {
  MapContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> mapContext = 
    new ChainMapContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>(
      context, rr, rw, conf);
  Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context mapperContext = 
    new WrappedMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>()
      .getMapContext(mapContext);
  return mapperContext;
}
 
Example #28
Source File: ColumnBasedHandlerTestUtil.java    From datawave with Apache License 2.0 5 votes vote down vote up
@Override
protected void flush(Multimap<BulkIngestKey,Value> entries, TaskInputOutputContext<?,?,BulkIngestKey,Value> context) throws IOException,
                InterruptedException {
    for (Map.Entry<BulkIngestKey,Value> entry : entries.entries()) {
        cache.put(entry.getKey(), entry.getValue());
    }
}
 
Example #29
Source File: LogUtil.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
 * Get desired information from reducer's context like job, reduce instance,
 * etc
 * 
 * @param context
 *            - reduce context to get all the required information of
 *            reducer like its job, reduce instance, etc
 * @param className
 *            - Class which is calling this method
 * @param methodName
 *            - Class Method which is calling this method
 */
@SuppressWarnings(RAW_TYPES)
public static void getReduceContextInfo(TaskInputOutputContext context,
		String className, String methodName) {
	Counter counter = context.getCounter(MAPRED_COUNTER,
			REDUCE_INPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());

	counter = context.getCounter(MAPRED_COUNTER, REDUCE_OUTPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());
}
 
Example #30
Source File: Chain.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Create a map context that is based on ChainMapContext and the given record
 * reader and record writer
 */
private <KEYIN, VALUEIN, KEYOUT, VALUEOUT> 
Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context createMapContext(
    RecordReader<KEYIN, VALUEIN> rr, RecordWriter<KEYOUT, VALUEOUT> rw,
    TaskInputOutputContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context,
    Configuration conf) {
  MapContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> mapContext = 
    new ChainMapContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>(
      context, rr, rw, conf);
  Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context mapperContext = 
    new WrappedMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>()
      .getMapContext(mapContext);
  return mapperContext;
}