Java Code Examples for org.apache.hadoop.mapred.Reporter#incrCounter()

The following examples show how to use org.apache.hadoop.mapred.Reporter#incrCounter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DistCh.java From hadoop with Apache License 2.0

6 votes

/** Run a FileOperation */
public void map(Text key, FileOperation value,
    OutputCollector<WritableComparable<?>, Text> out, Reporter reporter
    ) throws IOException {
  try {
    value.run(jobconf);
    ++succeedcount;
    reporter.incrCounter(Counter.SUCCEED, 1);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);

    String s = "FAIL: " + value + ", " + StringUtils.stringifyException(e);
    out.collect(null, new Text(s));
    LOG.info(s);
  } finally {
    reporter.setStatus(getCountString());
  }
}

Example 2

Source File: DistCh.java From RDFS with Apache License 2.0

6 votes

/** Run a FileOperation */
public void map(Text key, FileOperation value,
    OutputCollector<WritableComparable<?>, Text> out, Reporter reporter
    ) throws IOException {
  try {
    value.run(jobconf);
    ++succeedcount;
    reporter.incrCounter(Counter.SUCCEED, 1);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);

    String s = "FAIL: " + value + ", " + StringUtils.stringifyException(e);
    out.collect(null, new Text(s));
    LOG.info(s);
  } finally {
    reporter.setStatus(getCountString());
  }
}

Example 3

Source File: ReaderWriterProfiler.java From hive-dwrf with Apache License 2.0

6 votes

public static void log(Reporter logReporter) {
  for (Counter c : Counter.values()) {
    LOG.info(c + " start (" + started[c.value] + "), end (" + ended[c.value] + "): " +  profileTimes[c.value]);
    if (logReporter != null) {
      logReporter.incrCounter(c, profileTimes[c.value]);
    }
  }

  long read = profileTypeTimes[Counter.Type.READ.ordinal()];
  long write = profileTypeTimes[Counter.Type.WRITE.ordinal()];
  if (logReporter != null) {
    LOG.info("read time: " + read);
    LOG.info("write time: " + write);
    logReporter.incrCounter(ReadWriteCounter.READ_TIME, read);
    logReporter.incrCounter(ReadWriteCounter.WRITE_TIME, write);
  }
}

Example 4

Source File: WikiMetadata.java From wikireverse with MIT License

6 votes

public boolean isWikiPage(String url, Reporter reporter) throws URISyntaxException {
	boolean result = false;
	
	if (url != null && (url.indexOf(WIKIPEDIA_DOMAIN) >= 0 ||
			url.indexOf(WIKIMEDIA_DOMAIN) >= 0)) {
		URI pageUri = new URI(url);
		String pageHost = pageUri.getHost();
		
		if (pageHost != null && pageHost.endsWith(WIKIPEDIA_DOMAIN)) {
			LOG.info(url);
			reporter.incrCounter(COUNTER_GROUP, SKIP_WIKIPEDIA_PAGE, 1);
			result = true;
		}

		if (pageHost != null && pageHost.endsWith(WIKIMEDIA_DOMAIN)) {
			LOG.info(url);
			reporter.incrCounter(COUNTER_GROUP, SKIP_WIKIMEDIA_PAGE, 1);
			result = true;
		}
	}
	
	return result;
}

Example 5

Source File: SleepJob.java From RDFS with Apache License 2.0

6 votes

public void reduce(IntWritable key, Iterator<NullWritable> values,
    OutputCollector<NullWritable, NullWritable> output, Reporter reporter)
    throws IOException {
  List<String> counterNames = getCounterNames();
  for (String counterName : counterNames) {
    reporter.incrCounter("Counters from Reducers", counterName, 1);
  }
  try {
    reporter.setStatus("Sleeping... (" +
        (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
      Thread.sleep(reduceSleepDuration);
    
  }
  catch (InterruptedException ex) {
    throw (IOException)new IOException(
        "Interrupted while sleeping").initCause(ex);
  }
  count++;
}

Example 6

Source File: XMLIngestMapper.java From hadoop-solr with Apache License 2.0

6 votes

@Override
protected LWDocument[] toDocuments(
  Writable key,
  Text text,
  Reporter reporter,
  Configuration configuration) throws IOException {
  LWDocument[] docs = null;
  try {
    docs = toDocumentsImpl(key, text);
  } catch (Exception exc) {
    log.error("Failed to process XML " + key + " due to: " + exc, exc);
    reporter.incrCounter("XMLIngestMapper", "BadDocs", 1);
  }

  if (docs != null && docs.length > 0) {
    reporter.incrCounter("XMLIngestMapper", "DocsCreated", docs.length);
  } else {
    log.warn("No documents added in: " + key);
    docs = new LWDocument[0];
  }
  return docs;
}

Example 7

Source File: SleepJob.java From RDFS with Apache License 2.0

6 votes

public void map(IntWritable key, IntWritable value,
    OutputCollector<IntWritable, NullWritable> output, Reporter reporter)
    throws IOException {

  List<String> counterNames = getCounterNames();
  for (String counterName : counterNames) {
    reporter.incrCounter("Counters from Mappers", counterName, 1);
  }
  //it is expected that every map processes mapSleepCount number of records. 
  try {
    reporter.setStatus("Sleeping... (" +
        (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
    Thread.sleep(mapSleepDuration);
  }
  catch (InterruptedException ex) {
    throw (IOException)new IOException(
        "Interrupted while sleeping").initCause(ex);
  }
  ++count;
  // output reduceSleepCount * numReduce number of random values, so that
  // each reducer will get reduceSleepCount number of keys.
  int k = key.get();
  for (int i = 0; i < value.get(); ++i) {
    output.collect(new IntWritable(k + i), NullWritable.get());
  }
}

Example 8

Source File: WikiReverseMapper.java From wikireverse with MIT License

5 votes

public void run(RecordReader<LongWritable, WritableWarcRecord> input,
				OutputCollector<Text, LinkArrayWritable> output, Reporter reporter)
				throws IOException {
	try {
		WikiMetadata wikiMetadata = new WikiMetadata();
		
		LongWritable key = input.createKey();
		WritableWarcRecord value = input.createValue();
		
		while (input.next(key, value)) {
			map(key, value, output, reporter, wikiMetadata);
			reporter.incrCounter(COUNTER_GROUP, RECORDS_FETCHED, 1);
		}
		
	} catch(InterruptedException ie) {
		reporter.incrCounter(COUNTER_GROUP, MAPPER_INTERRUPTED, 1);
		LOG.error(StringUtils.stringifyException(ie));
	} catch(IOException io) {
		reporter.incrCounter(COUNTER_GROUP, RUN_IO_EXCEPTION, 1);
		LOG.error(StringUtils.stringifyException(io));
	} catch(Exception e) {
		reporter.incrCounter(COUNTER_GROUP, RUN_EXCEPTION, 1);
		LOG.error(StringUtils.stringifyException(e));
	} finally {
		input.close();
	}
}

Example 9

Source File: RemoteParForUtils.java From systemds with Apache License 2.0

5 votes

public static void incrementParForMRCounters(Reporter reporter, long deltaTasks, long deltaIterations)
{
	//report parfor counters
	if( deltaTasks>0 )
		reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMTASKS.toString(), deltaTasks);
	if( deltaIterations>0 )
		reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMITERS.toString(), deltaIterations);
	
	JobConf job = ConfigurationManager.getCachedJobConf();
	if( DMLScript.STATISTICS  && !InfrastructureAnalyzer.isLocalMode(job) ) 
	{
		//report cache statistics
		reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JITCOMPILE.toString(), Statistics.getJITCompileTime());
		reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_COUNT.toString(), Statistics.getJVMgcCount());
		reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_TIME.toString(), Statistics.getJVMgcTime());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_MEM.toString(), CacheStatistics.getMemHits());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FSBUFF.toString(), CacheStatistics.getFSBuffHits());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FS.toString(), CacheStatistics.getFSHits());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_HDFS.toString(), CacheStatistics.getHDFSHits());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FSBUFF.toString(), CacheStatistics.getFSBuffWrites());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FS.toString(), CacheStatistics.getFSWrites());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_HDFS.toString(), CacheStatistics.getHDFSWrites());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQR.toString(), CacheStatistics.getAcquireRTime());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQM.toString(), CacheStatistics.getAcquireMTime());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_RLS.toString(), CacheStatistics.getReleaseTime());
		reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_EXP.toString(), CacheStatistics.getExportTime());
	
		//reset cache statistics to prevent overlapping reporting
		CacheStatistics.reset();
	}
}

Example 10

Source File: LinkArrayReducer.java From wikireverse with MIT License

5 votes

public void reduce(Text key, Iterator<LinkArrayWritable> values, OutputCollector<Text, LinkArrayWritable> output, Reporter reporter) throws IOException {
	try {
		LinkArrayWritable value = new LinkArrayWritable();
		Writable[] allValues = new Writable[0];
		Writable[] combinedValues;
		Writable[] nextValues;
		
		while (values.hasNext()) {
			nextValues = values.next().get();
			combinedValues = new Writable[allValues.length + nextValues.length];
			
			System.arraycopy(allValues, 0, combinedValues, 0, allValues.length);
			System.arraycopy(nextValues, 0, combinedValues, allValues.length, nextValues.length);
			
			allValues = combinedValues;
		}
		
		value.set(allValues);
		output.collect(key, value);
		
		reporter.incrCounter(COUNTER_GROUP, URLS_REDUCED, 1);
		reporter.incrCounter(COUNTER_GROUP, RESULTS_COMBINED, allValues.length);		

	} catch (Exception e) {
		reporter.incrCounter(COUNTER_GROUP, REDUCE_EXCEPTION, 1);
		LOG.error(StringUtils.stringifyException(e));
	}
}

Example 11

Source File: SegmentCombinerMapper.java From wikireverse with MIT License

5 votes

public void map(Text key, LinkArrayWritable value, OutputCollector<Text, LinkArrayWritable> output, Reporter reporter)
        throws IOException {

	try {
		output.collect(key, value);

		reporter.incrCounter(COUNTER_GROUP, RECORDS_FETCHED, 1);
		reporter.incrCounter(COUNTER_GROUP, RESULTS_COUNTED, value.get().length);
		
	} catch (Exception e) {
		reporter.incrCounter(COUNTER_GROUP, MAP_EXCEPTION, 1);
		LOG.error(StringUtils.stringifyException(e));
	}
}

Example 12

Source File: TestIngestReducer.java From hadoop-solr with Apache License 2.0

5 votes

@Override
public void reduce(Text key, Iterator<LWDocumentWritable> values,
    OutputCollector<Text, LWDocumentWritable> output, Reporter reporter) throws IOException {
  count++;
  reduce(key, values, output, reporter);
  reporter.incrCounter("TestIngestReducer", "count", count);
}

Example 13

Source File: DynamoDBFibonacciRetryer.java From emr-dynamodb-connector with Apache License 2.0

5 votes

private void incrementRetryCounter(Reporter reporter, PrintCounter retryCounter) {
  if (reporter != null) {
    if (retryCounter != null) {
      reporter.incrCounter(retryCounter.getGroup(), retryCounter.getName(), 1);
    } else {
      reporter.progress();
    }
  }
}

Example 14

Source File: RandomTextWriter.java From hadoop-gpu with Apache License 2.0

5 votes

/**
 * Given an output filename, write a bunch of random records to it.
 */
public void map(Text key, Text value,
                OutputCollector<Text, Text> output, 
                Reporter reporter) throws IOException {
  int itemCount = 0;
  while (numBytesToWrite > 0) {
    // Generate the key/value 
    int noWordsKey = minWordsInKey + 
      (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0);
    int noWordsValue = minWordsInValue + 
      (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0);
    Text keyWords = generateSentence(noWordsKey);
    Text valueWords = generateSentence(noWordsValue);
    
    // Write the sentence 
    output.collect(keyWords, valueWords);
    
    numBytesToWrite -= (keyWords.getLength() + valueWords.getLength());
    
    // Update counters, progress etc.
    reporter.incrCounter(Counters.BYTES_WRITTEN, 
                         (keyWords.getLength()+valueWords.getLength()));
    reporter.incrCounter(Counters.RECORDS_WRITTEN, 1);
    if (++itemCount % 200 == 0) {
      reporter.setStatus("wrote record " + itemCount + ". " + 
                         numBytesToWrite + " bytes left.");
    }
  }
  reporter.setStatus("done with " + itemCount + " records.");
}

Example 15

Source File: RowCounter.java From hbase with Apache License 2.0

5 votes

public void map(ImmutableBytesWritable row, Result values,
    OutputCollector<ImmutableBytesWritable, Result> output,
    Reporter reporter)
throws IOException {
    // Count every row containing data, whether it's in qualifiers or values
    reporter.incrCounter(Counters.ROWS, 1);
}

Example 16

Source File: DistCp.java From RDFS with Apache License 2.0

5 votes

/** Map method. Copies one file from source file system to destination.
 * @param key src len
 * @param value FilePair (FileStatus src, Path dst)
 * @param out Log of failed copies
 * @param reporter
 */
public void map(LongWritable key,
                FilePairComparable value,
                OutputCollector<FilePairComparable, Text> out,
                Reporter reporter) throws IOException {
  final FileStatus srcstat = value.input;
  final Path relativedst = new Path(value.output);
  try {
    copy(value, out, reporter);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);
    updateStatus(reporter);
    final String sfailure = "FAIL " + relativedst + " : " +
                      StringUtils.stringifyException(e);
    out.collect(value, new Text(sfailure));
    LOG.info(sfailure);
    try {
      for (int i = 0; i < 3; ++i) {
        try {
          final Path tmp = new Path(attemptTmpRoot, relativedst);
          if (destFileSys.delete(tmp, true))
            break;
        } catch (Throwable ex) {
          // ignore, we are just cleaning up
          LOG.debug("Ignoring cleanup exception", ex);
        }
        // update status, so we don't get timed out
        updateStatus(reporter);
        Thread.sleep(3 * 1000);
      }
    } catch (InterruptedException inte) {
      throw (IOException)new IOException().initCause(inte);
    }
  } finally {
    updateStatus(reporter);
  }
}

Example 17

Source File: DeduplicationJob.java From nutch-htmlunit with Apache License 2.0

5 votes

private void writeOutAsDuplicate(CrawlDatum datum,
        OutputCollector<Text, CrawlDatum> output, Reporter reporter)
        throws IOException {
    datum.setStatus(CrawlDatum.STATUS_DB_DUPLICATE);
    Text key = (Text) datum.getMetaData().remove(urlKey);
    reporter.incrCounter("DeduplicationJobStatus",
            "Documents marked as duplicate", 1);
    output.collect(key, datum);
}

Example 18

Source File: DistCpV1.java From hadoop with Apache License 2.0

5 votes

/**
 * Copies single file to the path specified by tmpfile.
 * @param srcstat  src path and metadata
 * @param tmpfile  temporary file to which copy is to be done
 * @param absdst   actual destination path to which copy is to be done
 * @param reporter
 * @return Number of bytes copied
 */
private long doCopyFile(FileStatus srcstat, Path tmpfile, Path absdst,
                        Reporter reporter) throws IOException {
  long bytesCopied = 0L;
  Path srcPath = srcstat.getPath();
  // open src file
  try (FSDataInputStream in = srcPath.getFileSystem(job).open(srcPath)) {
    reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
    // open tmp file
    try (FSDataOutputStream out = create(tmpfile, reporter, srcstat)) {
      LOG.info("Copying file " + srcPath + " of size " +
               srcstat.getLen() + " bytes...");
    
      // copy file
      for(int bytesRead; (bytesRead = in.read(buffer)) >= 0; ) {
        out.write(buffer, 0, bytesRead);
        bytesCopied += bytesRead;
        reporter.setStatus(
            String.format("%.2f ", bytesCopied*100.0/srcstat.getLen())
            + absdst + " [ " +
            TraditionalBinaryPrefix.long2String(bytesCopied, "", 1) + " / "
            + TraditionalBinaryPrefix.long2String(srcstat.getLen(), "", 1)
            + " ]");
      }
    }
  }
  return bytesCopied;
}

Example 19

Source File: DistCpV1.java From hadoop with Apache License 2.0

5 votes

/**
 * Increment number of files copied and bytes copied and then report status
 */
void updateCopyStatus(FileStatus srcstat, Reporter reporter) {
  copycount++;
  reporter.incrCounter(Counter.BYTESCOPIED, srcstat.getLen());
  reporter.incrCounter(Counter.COPY, 1);
  updateStatus(reporter);
}

Example 20

Source File: DistCpV1.java From hadoop with Apache License 2.0

4 votes

/** Map method. Copies one file from source file system to destination.
 * @param key src len
 * @param value FilePair (FileStatus src, Path dst)
 * @param out Log of failed copies
 * @param reporter
 */
public void map(LongWritable key,
                FilePair value,
                OutputCollector<WritableComparable<?>, Text> out,
                Reporter reporter) throws IOException {
  final FileStatus srcstat = value.input;
  final Path relativedst = new Path(value.output);
  try {
    copyWithRetries(srcstat, relativedst, out, reporter);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);
    updateStatus(reporter);
    final String sfailure = "FAIL " + relativedst + " : " +
                      StringUtils.stringifyException(e);
    out.collect(null, new Text(sfailure));
    LOG.info(sfailure);
    if (e instanceof FileNotFoundException) {
      final String s = "Possible Cause for failure: Either the filesystem "
                       + srcstat.getPath().getFileSystem(job)
                       + " is not accessible or the file is deleted";
      LOG.error(s);
      out.collect(null, new Text(s));
    }

    try {
      for (int i = 0; i < 3; ++i) {
        try {
          final Path tmp = new Path(job.get(TMP_DIR_LABEL), relativedst);
          if (destFileSys.delete(tmp, true))
            break;
        } catch (Throwable ex) {
          // ignore, we are just cleaning up
          LOG.debug("Ignoring cleanup exception", ex);
        }
        // update status, so we don't get timed out
        updateStatus(reporter);
        Thread.sleep(3 * 1000);
      }
    } catch (InterruptedException inte) {
      throw (IOException)new IOException().initCause(inte);
    }
  } finally {
    updateStatus(reporter);
  }
}