Java Code Examples for org.apache.hadoop.metrics.MetricsContext

The following examples show how to use org.apache.hadoop.metrics.MetricsContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: LocalJobRunnerMetrics.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    metricsRecord.incrMetric("maps_launched", numMapTasksLaunched);
    metricsRecord.incrMetric("maps_completed", numMapTasksCompleted);
    metricsRecord.incrMetric("reduces_launched", numReduceTasksLaunched);
    metricsRecord.incrMetric("reduces_completed", numReduceTasksCompleted);
    metricsRecord.incrMetric("waiting_maps", numWaitingMaps);
    metricsRecord.incrMetric("waiting_reduces", numWaitingReduces);

    numMapTasksLaunched = 0;
    numMapTasksCompleted = 0;
    numReduceTasksLaunched = 0;
    numReduceTasksCompleted = 0;
    numWaitingMaps = 0;
    numWaitingReduces = 0;
  }
  metricsRecord.update();
}
 
Example 2
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
@InterfaceAudience.Private
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.parseInt(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 
Example 3
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
@InterfaceAudience.Private
@Override
protected void emitRecord(String contextName, String recordName,
    OutputRecord outRec) throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).emitRecord(
        contextName, recordName, outRec);
      if (contextName == null || recordName == null || outRec == null) {
        throw new IOException(contextName + ":" + recordName + ":" + outRec);
      }
    } catch (IOException e) {
      LOG.warn("emitRecord failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 4
Source Project: big-c   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
@InterfaceAudience.Private
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.parseInt(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 
Example 5
Source Project: big-c   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
@InterfaceAudience.Private
@Override
protected void emitRecord(String contextName, String recordName,
    OutputRecord outRec) throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).emitRecord(
        contextName, recordName, outRec);
      if (contextName == null || recordName == null || outRec == null) {
        throw new IOException(contextName + ":" + recordName + ":" + outRec);
      }
    } catch (IOException e) {
      LOG.warn("emitRecord failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 6
Source Project: RDFS   Source File: ReduceTask.java    License: Apache License 2.0 6 votes vote down vote up
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    shuffleMetrics.incrMetric("shuffle_input_bytes", numBytes);
    shuffleMetrics.incrMetric("shuffle_failed_fetches",
                              numFailedFetches);
    shuffleMetrics.incrMetric("shuffle_success_fetches",
                              numSuccessFetches);
    if (numCopiers != 0) {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent",
          100*((float)numThreadsBusy/numCopiers));
    } else {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent", 0);
    }
    numBytes = 0;
    numSuccessFetches = 0;
    numFailedFetches = 0;
  }
  shuffleMetrics.update();
}
 
Example 7
Source Project: RDFS   Source File: ProxyJobTracker.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void doUpdates(MetricsContext unused) {
  synchronized (aggregateJobStats) {
    // Update metrics with aggregate job stats and reset the aggregate.
    aggregateJobStats.incrementMetricsAndReset(metricsRecord);

    incrementMetricsAndReset(metricsRecord, aggregateCounters);

    for (Map.Entry<String, MetricsRecord> entry :
      poolToMetricsRecord.entrySet()) {
      String pool = entry.getKey();

      JobStats poolJobStats = poolToJobStats.get(pool);
      poolJobStats.incrementMetricsAndReset(entry.getValue());

      Counters poolCounters = poolToJobCounters.get(pool);
      incrementMetricsAndReset(entry.getValue(), poolCounters);
    }
  }
}
 
Example 8
Source Project: RDFS   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.valueOf(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 
Example 9
Source Project: hadoop-gpu   Source File: TaskTracker.java    License: Apache License 2.0 6 votes vote down vote up
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    if (workerThreads != 0) {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent", 
          100*((float)serverHandlerBusy/workerThreads));
    } else {
      shuffleMetricsRecord.setMetric("shuffle_handler_busy_percent", 0);
    }
    shuffleMetricsRecord.incrMetric("shuffle_output_bytes", 
                                    outputBytes);
    shuffleMetricsRecord.incrMetric("shuffle_failed_outputs", 
                                    failedOutputs);
    shuffleMetricsRecord.incrMetric("shuffle_success_outputs", 
                                    successOutputs);
    outputBytes = 0;
    failedOutputs = 0;
    successOutputs = 0;
  }
  shuffleMetricsRecord.update();
}
 
Example 10
Source Project: hadoop-gpu   Source File: ReduceTask.java    License: Apache License 2.0 6 votes vote down vote up
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    shuffleMetrics.incrMetric("shuffle_input_bytes", numBytes);
    shuffleMetrics.incrMetric("shuffle_failed_fetches", 
                              numFailedFetches);
    shuffleMetrics.incrMetric("shuffle_success_fetches", 
                              numSuccessFetches);
    if (numCopiers != 0) {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent",
          100*((float)numThreadsBusy/numCopiers));
    } else {
      shuffleMetrics.setMetric("shuffle_fetchers_busy_percent", 0);
    }
    numBytes = 0;
    numSuccessFetches = 0;
    numFailedFetches = 0;
  }
  shuffleMetrics.update();
}
 
Example 11
Source Project: RDFS   Source File: HighTideNodeMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public HighTideNodeMetrics(Configuration conf, HighTideNode hightideNode) {
  String sessionId = conf.get("session.id");
  // Initiate Java VM metrics
  JvmMetrics.init("HighTideNode", sessionId);


  // Now the Mbean for the name node - this also registers the MBean
  hightidenodeActivityMBean = new HighTideNodeActivityMBean(registry);

  // Create a record for HighTideNode metrics
  MetricsContext metricsContext = MetricsUtil.getContext("dfs");
  metricsRecord = MetricsUtil.createRecord(metricsContext, "hightidenode");
  metricsRecord.setTag("sessionId", sessionId);
  metricsContext.registerUpdater(this);
  LOG.info("Initializing HighTideNodeMetrics using context object:" +
            metricsContext.getClass().getName());
}
 
Example 12
Source Project: incubator-tez   Source File: ShuffleClientMetrics.java    License: Apache License 2.0 6 votes vote down vote up
ShuffleClientMetrics(String dagName, String vertexName, int taskIndex, Configuration conf, 
    String user) {
  this.numCopiers = 
      conf.getInt(
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 
          TezJobConfig.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT);

  MetricsContext metricsContext = MetricsUtil.getContext(Constants.TEZ);
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", user);
  this.shuffleMetrics.setTag("dagName", dagName);
  this.shuffleMetrics.setTag("taskId", TezRuntimeUtils.getTaskIdentifier(vertexName, taskIndex));
  this.shuffleMetrics.setTag("sessionId", 
      conf.get(
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID,
          TezRuntimeFrameworkConfigs.TEZ_RUNTIME_METRICS_SESSION_ID_DEFAULT));
  metricsContext.registerUpdater(this);
}
 
Example 13
Source Project: hadoop-gpu   Source File: CompositeContext.java    License: Apache License 2.0 6 votes vote down vote up
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);
  int nKids;
  try {
    String sKids = getAttribute(ARITY_LABEL);
    nKids = Integer.valueOf(sKids);
  } catch (Exception e) {
    LOG.error("Unable to initialize composite metric " + contextName +
              ": could not init arity", e);
    return;
  }
  for (int i = 0; i < nKids; ++i) {
    MetricsContext ctxt = MetricsUtil.getContext(
        String.format(SUB_FMT, contextName, i), contextName);
    if (null != ctxt) {
      subctxt.add(ctxt);
    }
  }
}
 
Example 14
Source Project: hadoop-gpu   Source File: TaskTrackerMetricsInst.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Since this object is a registered updater, this method will be called
 * periodically, e.g. every 5 seconds.
 */
@Override
public void doUpdates(MetricsContext unused) {
  synchronized (this) {
    metricsRecord.setMetric("maps_running", tt.mapTotal);
    metricsRecord.setMetric("reduces_running", tt.reduceTotal);
    metricsRecord.setMetric("mapTaskSlots", (short)tt.getMaxCurrentMapTasks());
    metricsRecord.setMetric("reduceTaskSlots", 
                                 (short)tt.getMaxCurrentReduceTasks());
    metricsRecord.incrMetric("tasks_completed", numCompletedTasks);
    metricsRecord.incrMetric("tasks_failed_timeout", timedoutTasks);
    metricsRecord.incrMetric("tasks_failed_ping", tasksFailedPing);
    
    numCompletedTasks = 0;
    timedoutTasks = 0;
    tasksFailedPing = 0;
  }
    metricsRecord.update();
}
 
Example 15
Source Project: hadoop   Source File: LocalJobRunnerMetrics.java    License: Apache License 2.0 5 votes vote down vote up
public LocalJobRunnerMetrics(JobConf conf) {
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  // record name is jobtracker for compatibility 
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 
Example 16
Source Project: hadoop   Source File: ShuffleClientMetrics.java    License: Apache License 2.0 5 votes vote down vote up
ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf) {
  this.numCopiers = jobConf.getInt(MRJobConfig.SHUFFLE_PARALLEL_COPIES, 5);

  MetricsContext metricsContext = MetricsUtil.getContext("mapred");
  this.shuffleMetrics = 
    MetricsUtil.createRecord(metricsContext, "shuffleInput");
  this.shuffleMetrics.setTag("user", jobConf.getUser());
  this.shuffleMetrics.setTag("jobName", jobConf.getJobName());
  this.shuffleMetrics.setTag("jobId", reduceId.getJobID().toString());
  this.shuffleMetrics.setTag("taskId", reduceId.toString());
  this.shuffleMetrics.setTag("sessionId", jobConf.getSessionId());
  metricsContext.registerUpdater(this);
}
 
Example 17
Source Project: hadoop-gpu   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void flush() throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).flush();
    } catch (IOException e) {
      LOG.warn("flush failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 18
Source Project: hadoop   Source File: JvmMetrics.java    License: Apache License 2.0 5 votes vote down vote up
/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 
Example 19
Source Project: hadoop-gpu   Source File: JvmMetrics.java    License: Apache License 2.0 5 votes vote down vote up
/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}
 
Example 20
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
protected void flush() throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ((AbstractMetricsContext)ctxt).flush();
    } catch (IOException e) {
      LOG.warn("flush failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 21
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
public void startMonitoring() throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ctxt.startMonitoring();
    } catch (IOException e) {
      LOG.warn("startMonitoring failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 22
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
public void stopMonitoring() {
  for (MetricsContext ctxt : subctxt) {
    ctxt.stopMonitoring();
  }
}
 
Example 23
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return true if all subcontexts are monitoring.
 */
@InterfaceAudience.Private
@Override
public boolean isMonitoring() {
  boolean ret = true;
  for (MetricsContext ctxt : subctxt) {
    ret &= ctxt.isMonitoring();
  }
  return ret;
}
 
Example 24
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
public void close() {
  for (MetricsContext ctxt : subctxt) {
    ctxt.close();
  }
}
 
Example 25
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
public void registerUpdater(Updater updater) {
  for (MetricsContext ctxt : subctxt) {
    ctxt.registerUpdater(updater);
  }
}
 
Example 26
Source Project: hadoop   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
@Override
public void unregisterUpdater(Updater updater) {
  for (MetricsContext ctxt : subctxt) {
    ctxt.unregisterUpdater(updater);
  }
}
 
Example 27
Source Project: big-c   Source File: LocalJobRunnerMetrics.java    License: Apache License 2.0 5 votes vote down vote up
public LocalJobRunnerMetrics(JobConf conf) {
  String sessionId = conf.getSessionId();
  // Initiate JVM Metrics
  JvmMetrics.init("JobTracker", sessionId);
  // Create a record for map-reduce metrics
  MetricsContext context = MetricsUtil.getContext("mapred");
  // record name is jobtracker for compatibility 
  metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
  metricsRecord.setTag("sessionId", sessionId);
  context.registerUpdater(this);
}
 
Example 28
Source Project: hadoop-gpu   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void startMonitoring() throws IOException {
  for (MetricsContext ctxt : subctxt) {
    try {
      ctxt.startMonitoring();
    } catch (IOException e) {
      LOG.warn("startMonitoring failed: " + ctxt.getContextName(), e);
    }
  }
}
 
Example 29
Source Project: hadoop-gpu   Source File: CompositeContext.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return true if all subcontexts are monitoring.
 */
@Override
public boolean isMonitoring() {
  boolean ret = true;
  for (MetricsContext ctxt : subctxt) {
    ret &= ctxt.isMonitoring();
  }
  return ret;
}
 
Example 30
Source Project: big-c   Source File: JvmMetrics.java    License: Apache License 2.0 5 votes vote down vote up
/** Creates a new instance of JvmMetrics */
private JvmMetrics(String processName, String sessionId,
  String recordName) {
    MetricsContext context = MetricsUtil.getContext("jvm");
    metrics = MetricsUtil.createRecord(context, recordName);
    metrics.setTag("processName", processName);
    metrics.setTag("sessionId", sessionId);
    context.registerUpdater(this);
}