org.apache.hadoop.mapred.Counters Java Examples

The following examples show how to use org.apache.hadoop.mapred.Counters. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
 
Example #2
Source File: XGBoostTrainUDTF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);
    }
    return booster;
}
 
Example #3
Source File: TestFetcher.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example #4
Source File: MergeManagerImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
 
Example #5
Source File: TestFetcher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example #6
Source File: JobHistoryHelper.java    From hiped2 with Apache License 2.0 6 votes vote down vote up
public static String extractCounter(String counterFromHist,
                                    String... counterNames)
    throws ParseException {
  Counters counters =
      Counters.fromEscapedCompactString(counterFromHist);
  for (Counters.Group group : counters) {
    for (Counters.Counter counter : group) {
      for (String counterName : counterNames) {
        if (counterName.equals(counter.getName())) {
          return String.valueOf(counter.getCounter());
        }
      }
    }
  }
  return null;
}
 
Example #7
Source File: ExtractJobMetrics.java    From hiped2 with Apache License 2.0 6 votes vote down vote up
public static String extractCounter(String counterFromHist,
                                    String... counterNames)
    throws ParseException {
  Counters counters =
      Counters.fromEscapedCompactString(counterFromHist);
  for (Counters.Group group : counters) {
    for (Counters.Counter counter : group) {
      for (String counterName : counterNames) {
        if (counterName.equals(counter.getName())) {
          return String.valueOf(counter.getCounter());
        }
      }
    }
  }
  return "";
}
 
Example #8
Source File: MRPigStatsUtil.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the count for the given counter name in the counter group
 * 'MultiStoreCounters'
 *
 * @param job the MR job
 * @param jobClient the Hadoop job client
 * @param counterName the counter name
 * @return the count of the given counter name
 */
public static long getMultiStoreCount(Job job, JobClient jobClient,
        String counterName) {
    long value = -1;
    try {
        RunningJob rj = jobClient.getJob(job.getAssignedJobID());
        if (rj != null) {
            Counters.Counter counter = rj.getCounters().getGroup(
                    MULTI_STORE_COUNTER_GROUP).getCounterForName(counterName);
            value = counter.getValue();
        }
    } catch (IOException e) {
        LOG.warn("Failed to get the counter for " + counterName, e);
    }
    return value;
}
 
Example #9
Source File: ShuffleSchedulerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ShuffleSchedulerImpl(JobConf job, TaskStatus status,
                        TaskAttemptID reduceId,
                        ExceptionReporter reporter,
                        Progress progress,
                        Counters.Counter shuffledMapsCounter,
                        Counters.Counter reduceShuffleBytes,
                        Counters.Counter failedShuffleCounter) {
  totalMaps = job.getNumMapTasks();
  abortFailureLimit = Math.max(30, totalMaps / 10);
  copyTimeTracker = new CopyTimeTracker();
  remainingMaps = totalMaps;
  finishedMaps = new boolean[remainingMaps];
  this.reporter = reporter;
  this.status = status;
  this.reduceId = reduceId;
  this.progress = progress;
  this.shuffledMapsCounter = shuffledMapsCounter;
  this.reduceShuffleBytes = reduceShuffleBytes;
  this.failedShuffleCounter = failedShuffleCounter;
  this.startTime = Time.monotonicNow();
  lastProgressTime = startTime;
  referee.start();
  this.maxFailedUniqueFetches = Math.min(totalMaps, 5);
  this.maxFetchFailuresBeforeReporting = job.getInt(
      MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT);
  this.reportReadErrorImmediately = job.getBoolean(
      MRJobConfig.SHUFFLE_NOTIFY_READERROR, true);

  this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY);
  this.maxHostFailures = job.getInt(
      MRJobConfig.MAX_SHUFFLE_FETCH_HOST_FAILURES,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES);
}
 
Example #10
Source File: TestStreamingCounters.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void validateCounters() throws IOException {
  Counters counters = job.running_.getCounters();
  assertNotNull("Counters", counters);
  Group group = counters.getGroup("UserCounters");
  assertNotNull("Group", group);
  Counter counter = group.getCounterForName("InputLines");
  assertNotNull("Counter", counter);
  assertEquals(3, counter.getCounter());
}
 
Example #11
Source File: TestStreamingCombiner.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testCommandLine() throws Exception {
  super.testCommandLine();
  // validate combiner counters
  String counterGrp = "org.apache.hadoop.mapred.Task$Counter";
  Counters counters = job.running_.getCounters();
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_INPUT_RECORDS").getValue() != 0);
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_OUTPUT_RECORDS").getValue() != 0);
}
 
Example #12
Source File: TestMiniCoronaSpeculativeTask.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private void verifyLaunchedTasks(SleepJob sleepJob, int maps, int reduces)
    throws IOException {
  Counters jobCounters = sleepJob.getRunningJob().getCounters();
  long launchedMaps = jobCounters.findCounter(
      JobInProgress.Counter.TOTAL_LAUNCHED_MAPS).getValue();
  long launchedReduces = jobCounters.findCounter(
      JobInProgress.Counter.TOTAL_LAUNCHED_REDUCES).getValue();
  Assert.assertEquals(maps, launchedMaps);
  Assert.assertEquals(reduces, launchedReduces);
}
 
Example #13
Source File: OutputHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void incrementCounter(int id, long amount) throws IOException {
  if (id < registeredCounters.size()) {
    Counters.Counter counter = registeredCounters.get(id);
    counter.increment(amount);
  } else {
    throw new IOException("Invalid counter with id: " + id);
  }
}
 
Example #14
Source File: JobMonitor.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private String toRaidProgressHtmlRow(String dateStr, Counters ctrs) {
  StringBuilder sb = new StringBuilder();
  sb.append(td(dateStr));
  sb.append(td(Long.toString(ctrs.getCounter(Counter.FILES_SUCCEEDED))));
  sb.append(td(StringUtils.humanReadableInt(ctrs.getCounter(Counter.PROCESSED_SIZE))));
  sb.append(td(StringUtils.humanReadableInt(ctrs.getCounter(Counter.SAVING_SIZE))));
  return tr(sb.toString());
}
 
Example #15
Source File: ShuffleSchedulerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ShuffleSchedulerImpl(JobConf job, TaskStatus status,
                        TaskAttemptID reduceId,
                        ExceptionReporter reporter,
                        Progress progress,
                        Counters.Counter shuffledMapsCounter,
                        Counters.Counter reduceShuffleBytes,
                        Counters.Counter failedShuffleCounter) {
  totalMaps = job.getNumMapTasks();
  abortFailureLimit = Math.max(30, totalMaps / 10);
  copyTimeTracker = new CopyTimeTracker();
  remainingMaps = totalMaps;
  finishedMaps = new boolean[remainingMaps];
  this.reporter = reporter;
  this.status = status;
  this.reduceId = reduceId;
  this.progress = progress;
  this.shuffledMapsCounter = shuffledMapsCounter;
  this.reduceShuffleBytes = reduceShuffleBytes;
  this.failedShuffleCounter = failedShuffleCounter;
  this.startTime = Time.monotonicNow();
  lastProgressTime = startTime;
  referee.start();
  this.maxFailedUniqueFetches = Math.min(totalMaps, 5);
  this.maxFetchFailuresBeforeReporting = job.getInt(
      MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT);
  this.reportReadErrorImmediately = job.getBoolean(
      MRJobConfig.SHUFFLE_NOTIFY_READERROR, true);

  this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY);
  this.maxHostFailures = job.getInt(
      MRJobConfig.MAX_SHUFFLE_FETCH_HOST_FAILURES,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES);
}
 
Example #16
Source File: TezDAGStats.java    From spork with Apache License 2.0 5 votes vote down vote up
private Counters covertToHadoopCounters(TezCounters tezCounters) {
    Counters counters = new Counters();
    for (CounterGroup tezGrp : tezCounters) {
        Group grp = counters.addGroup(tezGrp.getName(), tezGrp.getDisplayName());
        for (TezCounter counter : tezGrp) {
            grp.addCounter(counter.getName(), counter.getDisplayName(), counter.getValue());
        }
    }
    return counters;
}
 
Example #17
Source File: TestStreamingCounters.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void validateCounters() throws IOException {
  Counters counters = job.running_.getCounters();
  assertNotNull("Counters", counters);
  Group group = counters.getGroup("UserCounters");
  assertNotNull("Group", group);
  Counter counter = group.getCounterForName("InputLines");
  assertNotNull("Counter", counter);
  assertEquals(3, counter.getCounter());
}
 
Example #18
Source File: TestStreamingCombiner.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testCommandLine() throws Exception {
  super.testCommandLine();
  // validate combiner counters
  String counterGrp = "org.apache.hadoop.mapred.Task$Counter";
  Counters counters = job.running_.getCounters();
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_INPUT_RECORDS").getValue() != 0);
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_OUTPUT_RECORDS").getValue() != 0);
}
 
Example #19
Source File: MRTaskReporter.java    From tez with Apache License 2.0 5 votes vote down vote up
public Counters.Counter getCounter(String group, String name) {
  TezCounter counter = context.getCounters().findCounter(group, name);
  MRCounters.MRCounter mrCounter = null;
  if (counter != null) {
    mrCounter = new MRCounters.MRCounter(counter);
  }
  return mrCounter;
}
 
Example #20
Source File: VespaStorageTest.java    From vespa with Apache License 2.0 5 votes vote down vote up
private void assertAllDocumentsOk(String script, Configuration conf) throws Exception {
    PigServer ps = setup(script, conf);
    List<ExecJob> jobs = ps.executeBatch();
    PigStats stats = jobs.get(0).getStatistics();
    for (JobStats js : stats.getJobGraph()) {
        Counters hadoopCounters = ((MRJobStats)js).getHadoopCounters();
        assertNotNull(hadoopCounters);
        VespaCounters counters = VespaCounters.get(hadoopCounters);
        assertEquals(10, counters.getDocumentsSent());
        assertEquals(0, counters.getDocumentsFailed());
        assertEquals(10, counters.getDocumentsOk());
    }
}
 
Example #21
Source File: ReporterImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public Counter getCounter(String group, String name)
{
  Counters.Counter counter = null;
  if (counters != null) {
    counter = counters.findCounter(group, name);
  }

  return counter;
}
 
Example #22
Source File: ReporterImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public Counter getCounter(Enum<?> key)
{
  Counters.Counter counter = null;
  if (counters != null) {
    counter = counters.findCounter(key);
  }

  return counter;
}
 
Example #23
Source File: ReduceOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  reporter = new ReporterImpl(ReporterImpl.ReporterType.Reducer, new Counters());
  if (context != null) {
    operatorId = context.getId();
  }
  cacheObject = new HashMap<K1, List<V1>>();
  outputCollector = new OutputCollectorImpl<K2, V2>();
  if (reduceClass != null) {
    try {
      reduceObj = reduceClass.newInstance();
    } catch (Exception e) {
      logger.info("can't instantiate object {}", e.getMessage());
      throw new RuntimeException(e);
    }
    Configuration conf = new Configuration();
    InputStream stream = null;
    if (configFile != null && configFile.length() > 0) {
      logger.info("system /{}", configFile);
      stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
      if (stream == null) {
        logger.info("system {}", configFile);
        stream = ClassLoader.getSystemResourceAsStream(configFile);
      }
    }
    if (stream != null) {
      logger.info("found our stream... so adding it");
      conf.addResource(stream);
    }
    reduceObj.configure(new JobConf(conf));
  }

}
 
Example #24
Source File: HadoopShims.java    From spork with Apache License 2.0 5 votes vote down vote up
public static Counters getCounters(Job job) throws IOException {
    try {
        Cluster cluster = new Cluster(job.getJobConf());
        org.apache.hadoop.mapreduce.Job mrJob = cluster.getJob(job.getAssignedJobID());
        if (mrJob == null) { // In local mode, mrJob will be null
            mrJob = job.getJob();
        }
        return new Counters(mrJob.getCounters());
    } catch (Exception ir) {
        throw new IOException(ir);
    }
}
 
Example #25
Source File: HadoopJobHistoryLoader.java    From spork with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
    try {
        Counters counterGroups = Counters.fromEscapedCompactString(counters);
        for (Group otherGroup : counterGroups) {
            Group group = counterGroups.getGroup(otherGroup.getName());
            for (Counter otherCounter : otherGroup) {
                Counter counter = group.getCounterForName(otherCounter.getName());
                job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
            }
        }
    } catch (ParseException e) {
       LOG.warn("Failed to parse job counters", e);
    }
}
 
Example #26
Source File: HadoopRunningJobReporter.java    From CloverETL-Engine with GNU Lesser General Public License v2.1 5 votes vote down vote up
@Override
public List<HadoopCounterKeyValuePair> getAllCounters() throws IOException {
	List<HadoopCounterKeyValuePair> result = new ArrayList<HadoopCounterKeyValuePair>();
	for (Counters.Group group : job.getCounters()) {
		for (Counter counter : group) {
			result.add(new HadoopCounterKeyValuePair(counter.getName(), group.getName(), counter.getValue()));
		}
	}
	return result;
}
 
Example #27
Source File: HadoopCounterReporterTest.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public void setUp() throws Exception {

  String contextName = CONTEXT_NAME + "_" + UUID.randomUUID().toString();

  Reporter mockedReporter = Mockito.mock(Reporter.class);

  this.recordsProcessedCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName())))
      .thenReturn(this.recordsProcessedCount);

  this.recordProcessRateCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName())))
      .thenReturn(this.recordProcessRateCount);

  this.recordSizeDistributionCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName())))
      .thenReturn(this.recordSizeDistributionCount);

  this.totalDurationCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName())))
      .thenReturn(this.totalDurationCount);

  this.queueSize = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(contextName, QUEUE_SIZE)).thenReturn(this.queueSize);

  this.hadoopCounterReporter = HadoopCounterReporter.builder(mockedReporter)
      .convertRatesTo(TimeUnit.SECONDS)
      .convertDurationsTo(TimeUnit.SECONDS)
      .filter(MetricFilter.ALL)
      .build(MetricContext.builder(contextName).buildStrict());
}
 
Example #28
Source File: MRTaskReporter.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public Counters.Counter getCounter(Enum<?> name) {
  TezCounter counter = context.getCounters().findCounter(name);
  MRCounters.MRCounter mrCounter = null;
  if (counter != null) {
    mrCounter = new MRCounters.MRCounter(counter);
  }
  return mrCounter;
}
 
Example #29
Source File: MapRedCounterLoader.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Override
public ICounter getCounterByNameAndFlag(String groupName, String counterName, String counterFlag) {
  if (conf.getBoolean(counterFlag, true)) {
    Counters.Counter counter = reporter.getCounter(groupName, counterName);
    if (counter != null) {
      return new MapRedCounterAdapter(reporter.getCounter(groupName, counterName));
    }
  }
  return new BenchmarkCounter.NullCounter();
  }
 
Example #30
Source File: HadoopJob.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Print this job counters (for debugging purpose)
 */
void printCounters() {
  System.out.printf("New Job:\n", counters);
  for (String groupName : counters.getGroupNames()) {
    Counters.Group group = counters.getGroup(groupName);
    System.out.printf("\t%s[%s]\n", groupName, group.getDisplayName());

    for (Counters.Counter counter : group) {
      System.out.printf("\t\t%s: %s\n", counter.getDisplayName(),
                                        counter.getCounter());
    }
  }
  System.out.printf("\n");
}