org.apache.hadoop.metrics2.lib.MetricsRegistry Java Examples

The following examples show how to use org.apache.hadoop.metrics2.lib.MetricsRegistry. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ContainerMetrics.java    From big-c with Apache License 2.0 6 votes vote down vote up
ContainerMetrics(
    MetricsSystem ms, ContainerId containerId, long flushPeriodMs) {
  this.recordInfo =
      info(sourceName(containerId), RECORD_INFO.description());
  this.registry = new MetricsRegistry(recordInfo);
  this.metricsSystem = ms;
  this.containerId = containerId;
  this.flushPeriodMs = flushPeriodMs;
  scheduleTimerTaskIfRequired();

  this.pMemMBsStat = registry.newStat(
      PMEM_USAGE_METRIC_NAME, "Physical memory stats", "Usage", "MBs", true);
  this.cpuCoreUsagePercent = registry.newStat(
      PHY_CPU_USAGE_METRIC_NAME, "Physical Cpu core percent usage stats",
      "Usage", "Percents", true);
  this.milliVcoresUsed = registry.newStat(
      VCORE_USAGE_METRIC_NAME, "1000 times Vcore usage", "Usage",
      "MilliVcores", true);
  this.pMemLimitMbs = registry.newGauge(
      PMEM_LIMIT_METRIC_NAME, "Physical memory limit in MBs", 0);
  this.vMemLimitMbs = registry.newGauge(
      VMEM_LIMIT_METRIC_NAME, "Virtual memory limit in MBs", 0);
  this.cpuVcoreLimit = registry.newGauge(
      VCORE_LIMIT_METRIC_NAME, "CPU limit in number of vcores", 0);
}
 
Example #2
Source File: HadoopMetrics2Reporter.java    From kylin with Apache License 2.0 6 votes vote down vote up
private HadoopMetrics2Reporter(MetricRegistry registry, TimeUnit rateUnit, TimeUnit durationUnit,
        MetricFilter filter, MetricsSystem metrics2System, String jmxContext, String description, String recordName,
        String context) {
    super(registry, "hadoop-metrics2-reporter", filter, rateUnit, durationUnit);
    this.metrics2Registry = new MetricsRegistry(Interns.info(jmxContext, description));
    this.metrics2System = metrics2System;
    this.recordName = recordName;
    this.context = context;

    // These could really be Collection.emptyMap(), but this makes testing a bit easier.
    this.dropwizardGauges = EMPTY_GAUGE_MAP;
    this.dropwizardCounters = EMPTY_COUNTER_MAP;
    this.dropwizardHistograms = EMPTY_HISTOGRAM_MAP;
    this.dropwizardMeters = EMPTY_METER_MAP;
    this.dropwizardTimers = EMPTY_TIMER_MAP;

    // Register this source with the Metrics2 system.
    // Make sure this is the last thing done as getMetrics() can be called at any time after.
    this.metrics2System.register(Objects.requireNonNull(jmxContext), Objects.requireNonNull(description), this);
}
 
Example #3
Source File: XceiverClientMetrics.java    From hadoop-ozone with Apache License 2.0 6 votes vote down vote up
public void init() {
  int numEnumEntries = ContainerProtos.Type.values().length;
  this.registry = new MetricsRegistry(SOURCE_NAME);

  this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
  this.opsArray = new MutableCounterLong[numEnumEntries];
  this.containerOpsLatency = new MutableRate[numEnumEntries];
  for (int i = 0; i < numEnumEntries; i++) {
    pendingOpsArray[i] = registry.newCounter(
        "numPending" + ContainerProtos.Type.forNumber(i + 1),
        "number of pending" + ContainerProtos.Type.forNumber(i + 1) + " ops",
        (long) 0);
    opsArray[i] = registry
        .newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1),
            "number of" + ContainerProtos.Type.forNumber(i + 1) + " ops",
            (long) 0);

    containerOpsLatency[i] = registry.newRate(
        ContainerProtos.Type.forNumber(i + 1) + "Latency",
        "latency of " + ContainerProtos.Type.forNumber(i + 1)
        + " ops");
  }
}
 
Example #4
Source File: HadoopMetrics2Reporter.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private HadoopMetrics2Reporter(MetricRegistry registry, TimeUnit rateUnit, TimeUnit durationUnit,
        MetricFilter filter, MetricsSystem metrics2System, String jmxContext, String description, String recordName,
        String context) {
    super(registry, "hadoop-metrics2-reporter", filter, rateUnit, durationUnit);
    this.metrics2Registry = new MetricsRegistry(Interns.info(jmxContext, description));
    this.metrics2System = metrics2System;
    this.recordName = recordName;
    this.context = context;

    // These could really be Collection.emptyMap(), but this makes testing a bit easier.
    this.dropwizardGauges = EMPTY_GAUGE_MAP;
    this.dropwizardCounters = EMPTY_COUNTER_MAP;
    this.dropwizardHistograms = EMPTY_HISTOGRAM_MAP;
    this.dropwizardMeters = EMPTY_METER_MAP;
    this.dropwizardTimers = EMPTY_TIMER_MAP;

    // Register this source with the Metrics2 system.
    // Make sure this is the last thing done as getMetrics() can be called at any time after.
    this.metrics2System.register(Objects.requireNonNull(jmxContext), Objects.requireNonNull(description), this);
}
 
Example #5
Source File: QueueMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected QueueMetrics(MetricsSystem ms, String queueName, Queue parent, 
      boolean enableUserMetrics, Configuration conf) {
  registry = new MetricsRegistry(RECORD_INFO);
  this.queueName = queueName;
  this.parent = parent != null ? parent.getMetrics() : null;
  this.users = enableUserMetrics ? new HashMap<String, QueueMetrics>()
                                 : null;
  metricsSystem = ms;
  this.conf = conf;
  runningTime = buildBuckets(conf);
}
 
Example #6
Source File: ClusterMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void registerMetrics() {
  registry = new MetricsRegistry(RECORD_INFO);
  registry.tag(RECORD_INFO, "ResourceManager");
  MetricsSystem ms = DefaultMetricsSystem.instance();
  if (ms != null) {
    ms.register("ClusterMetrics", "Metrics for the Yarn Cluster", INSTANCE);
  }
}
 
Example #7
Source File: QueueMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected QueueMetrics(MetricsSystem ms, String queueName, Queue parent, 
      boolean enableUserMetrics, Configuration conf) {
  registry = new MetricsRegistry(RECORD_INFO);
  this.queueName = queueName;
  this.parent = parent != null ? parent.getMetrics() : null;
  this.users = enableUserMetrics ? new HashMap<String, QueueMetrics>()
                                 : null;
  metricsSystem = ms;
  this.conf = conf;
  runningTime = buildBuckets(conf);
}
 
Example #8
Source File: FSOpDurations.java    From big-c with Apache License 2.0 5 votes vote down vote up
private FSOpDurations() {
  registry = new MetricsRegistry(RECORD_INFO);
  registry.tag(RECORD_INFO, "FSOpDurations");

  MetricsSystem ms = DefaultMetricsSystem.instance();
  if (ms != null) {
    ms.register(RECORD_INFO.name(), RECORD_INFO.description(), this);
  }
}
 
Example #9
Source File: TestMetricsVisitor.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test the common use cases
 */
@Test public void testCommon() {
  MetricsVisitor visitor = mock(MetricsVisitor.class);
  MetricsRegistry registry = new MetricsRegistry("test");
  List<AbstractMetric> metrics = MetricsLists.builder("test")
      .addCounter(info("c1", "int counter"), 1)
      .addCounter(info("c2", "long counter"), 2L)
      .addGauge(info("g1", "int gauge"), 5)
      .addGauge(info("g2", "long gauge"), 6L)
      .addGauge(info("g3", "float gauge"), 7f)
      .addGauge(info("g4", "double gauge"), 8d)
      .metrics();

  for (AbstractMetric metric : metrics) {
    metric.visit(visitor);
  }

  verify(visitor).counter(c1.capture(), eq(1));
  assertEquals("c1 name", "c1", c1.getValue().name());
  assertEquals("c1 description", "int counter", c1.getValue().description());
  verify(visitor).counter(c2.capture(), eq(2L));
  assertEquals("c2 name", "c2", c2.getValue().name());
  assertEquals("c2 description", "long counter", c2.getValue().description());
  verify(visitor).gauge(g1.capture(), eq(5));
  assertEquals("g1 name", "g1", g1.getValue().name());
  assertEquals("g1 description", "int gauge", g1.getValue().description());
  verify(visitor).gauge(g2.capture(), eq(6L));
  assertEquals("g2 name", "g2", g2.getValue().name());
  assertEquals("g2 description", "long gauge", g2.getValue().description());
  verify(visitor).gauge(g3.capture(), eq(7f));
  assertEquals("g3 name", "g3", g3.getValue().name());
  assertEquals("g3 description", "float gauge", g3.getValue().description());
  verify(visitor).gauge(g4.capture(), eq(8d));
  assertEquals("g4 name", "g4", g4.getValue().name());
  assertEquals("g4 description", "double gauge", g4.getValue().description());
}
 
Example #10
Source File: RpcMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}
 
Example #11
Source File: RetryCacheMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up
RetryCacheMetrics(RetryCache retryCache) {
  name = "RetryCache."+ retryCache.getCacheName();
  registry = new MetricsRegistry(name);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Initialized "+ registry);
  }
}
 
Example #12
Source File: RetryCacheMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
RetryCacheMetrics(RetryCache retryCache) {
  name = "RetryCache."+ retryCache.getCacheName();
  registry = new MetricsRegistry(name);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Initialized "+ registry);
  }
}
 
Example #13
Source File: RpcMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
RpcMetrics(Server server, Configuration conf) {
  String port = String.valueOf(server.getListenerAddress().getPort());
  name = "RpcActivityForPort" + port;
  this.server = server;
  registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
  int[] intervals = conf.getInts(
      CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
  rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE,
      CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
  if (rpcQuantileEnable) {
    rpcQueueTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    rpcProcessingTimeMillisQuantiles =
        new MutableQuantiles[intervals.length];
    for (int i = 0; i < intervals.length; i++) {
      int interval = intervals[i];
      rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime"
          + interval + "s", "rpc queue time in milli second", "ops",
          "latency", interval);
      rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
          "rpcProcessingTime" + interval + "s",
          "rpc processing time in milli second", "ops", "latency", interval);
    }
  }
  LOG.debug("Initialized " + registry);
}
 
Example #14
Source File: FSOpDurations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private FSOpDurations() {
  registry = new MetricsRegistry(RECORD_INFO);
  registry.tag(RECORD_INFO, "FSOpDurations");

  MetricsSystem ms = DefaultMetricsSystem.instance();
  if (ms != null) {
    ms.register(RECORD_INFO.name(), RECORD_INFO.description(), this);
  }
}
 
Example #15
Source File: ClusterMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void registerMetrics() {
  registry = new MetricsRegistry(RECORD_INFO);
  registry.tag(RECORD_INFO, "ResourceManager");
  MetricsSystem ms = DefaultMetricsSystem.instance();
  if (ms != null) {
    ms.register("ClusterMetrics", "Metrics for the Yarn Cluster", INSTANCE);
  }
}
 
Example #16
Source File: ContainerMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
ContainerMetrics(
    MetricsSystem ms, ContainerId containerId, long flushPeriodMs) {
  this.recordInfo =
      info(sourceName(containerId), RECORD_INFO.description());
  this.registry = new MetricsRegistry(recordInfo);
  this.metricsSystem = ms;
  this.containerId = containerId;
  this.flushPeriodMs = flushPeriodMs;
  scheduleTimerTaskIfRequired();

  this.pMemMBsStat = registry.newStat(
      PMEM_USAGE_METRIC_NAME, "Physical memory stats", "Usage", "MBs", true);
  this.cpuCoreUsagePercent = registry.newStat(
      PHY_CPU_USAGE_METRIC_NAME, "Physical Cpu core percent usage stats",
      "Usage", "Percents", true);
  this.milliVcoresUsed = registry.newStat(
      VCORE_USAGE_METRIC_NAME, "1000 times Vcore usage", "Usage",
      "MilliVcores", true);
  this.pMemLimitMbs = registry.newGauge(
      PMEM_LIMIT_METRIC_NAME, "Physical memory limit in MBs", 0);
  this.vMemLimitMbs = registry.newGauge(
      VMEM_LIMIT_METRIC_NAME, "Virtual memory limit in MBs", 0);
  this.cpuVcoreLimit = registry.newGauge(
      VCORE_LIMIT_METRIC_NAME, "CPU limit in number of vcores", 0);
  this.gpuGcoreLimit = registry.newGauge(
      GCORE_LIMIT_METRIC_NAME, "GPU limit in number of gcores", 0);
}
 
Example #17
Source File: TestMetricsVisitor.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test the common use cases
 */
@Test public void testCommon() {
  MetricsVisitor visitor = mock(MetricsVisitor.class);
  MetricsRegistry registry = new MetricsRegistry("test");
  List<AbstractMetric> metrics = MetricsLists.builder("test")
      .addCounter(info("c1", "int counter"), 1)
      .addCounter(info("c2", "long counter"), 2L)
      .addGauge(info("g1", "int gauge"), 5)
      .addGauge(info("g2", "long gauge"), 6L)
      .addGauge(info("g3", "float gauge"), 7f)
      .addGauge(info("g4", "double gauge"), 8d)
      .metrics();

  for (AbstractMetric metric : metrics) {
    metric.visit(visitor);
  }

  verify(visitor).counter(c1.capture(), eq(1));
  assertEquals("c1 name", "c1", c1.getValue().name());
  assertEquals("c1 description", "int counter", c1.getValue().description());
  verify(visitor).counter(c2.capture(), eq(2L));
  assertEquals("c2 name", "c2", c2.getValue().name());
  assertEquals("c2 description", "long counter", c2.getValue().description());
  verify(visitor).gauge(g1.capture(), eq(5));
  assertEquals("g1 name", "g1", g1.getValue().name());
  assertEquals("g1 description", "int gauge", g1.getValue().description());
  verify(visitor).gauge(g2.capture(), eq(6L));
  assertEquals("g2 name", "g2", g2.getValue().name());
  assertEquals("g2 description", "long gauge", g2.getValue().description());
  verify(visitor).gauge(g3.capture(), eq(7f));
  assertEquals("g3 name", "g3", g3.getValue().name());
  assertEquals("g3 description", "float gauge", g3.getValue().description());
  verify(visitor).gauge(g4.capture(), eq(8d));
  assertEquals("g4 name", "g4", g4.getValue().name());
  assertEquals("g4 description", "double gauge", g4.getValue().description());
}
 
Example #18
Source File: ContainerMetrics.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
public ContainerMetrics(int[] intervals) {
  int numEnumEntries = ContainerProtos.Type.values().length;
  final int len = intervals.length;
  this.numOpsArray = new MutableCounterLong[numEnumEntries];
  this.opsBytesArray = new MutableCounterLong[numEnumEntries];
  this.opsLatency = new MutableRate[numEnumEntries];
  this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
  this.registry = new MetricsRegistry("StorageContainerMetrics");
  for (int i = 0; i < numEnumEntries; i++) {
    numOpsArray[i] = registry.newCounter(
        "num" + ContainerProtos.Type.forNumber(i + 1),
        "number of " + ContainerProtos.Type.forNumber(i + 1) + " ops",
        (long) 0);
    opsBytesArray[i] = registry.newCounter(
        "bytes" + ContainerProtos.Type.forNumber(i + 1),
        "bytes used by " + ContainerProtos.Type.forNumber(i + 1) + "op",
        (long) 0);
    opsLatency[i] = registry.newRate(
        "latency" + ContainerProtos.Type.forNumber(i + 1),
        ContainerProtos.Type.forNumber(i + 1) + " op");

    for (int j = 0; j < len; j++) {
      int interval = intervals[j];
      String quantileName = ContainerProtos.Type.forNumber(i + 1) + "Nanos"
          + interval + "s";
      opsLatQuantiles[i][j] = registry.newQuantiles(quantileName,
          "latency of Container ops", "ops", "latency", interval);
    }
  }
}
 
Example #19
Source File: CSMMetrics.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
public CSMMetrics() {
  int numCmdTypes = ContainerProtos.Type.values().length;
  this.opsLatency = new MutableRate[numCmdTypes];
  this.registry = new MetricsRegistry(CSMMetrics.class.getSimpleName());
  for (int i = 0; i < numCmdTypes; i++) {
    opsLatency[i] = registry.newRate(
        ContainerProtos.Type.forNumber(i + 1).toString(),
        ContainerProtos.Type.forNumber(i + 1) + " op");
  }
}
 
Example #20
Source File: TestMetricsSystemImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
TestSource(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #21
Source File: HadoopMetrics2Reporter.java    From kylin with Apache License 2.0 4 votes vote down vote up
MetricsRegistry getMetrics2Registry() {
    return metrics2Registry;
}
 
Example #22
Source File: TestMetricsSystemImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
TestSource2(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #23
Source File: RpcDetailedMetrics.java    From big-c with Apache License 2.0 4 votes vote down vote up
RpcDetailedMetrics(int port) {
  name = "RpcDetailedActivityForPort"+ port;
  registry = new MetricsRegistry("rpcdetailed")
      .tag("port", "RPC port", String.valueOf(port));
  LOG.debug(registry.info());
}
 
Example #24
Source File: TestGangliaMetrics.java    From big-c with Apache License 2.0 4 votes vote down vote up
TestSource(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #25
Source File: TestMetricsSourceAdapter.java    From big-c with Apache License 2.0 4 votes vote down vote up
TestSource(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #26
Source File: ClientSCMMetrics.java    From big-c with Apache License 2.0 4 votes vote down vote up
private ClientSCMMetrics() {
  registry = new MetricsRegistry("clientRequests");
  LOG.debug("Initialized " + registry);
}
 
Example #27
Source File: SharedCacheUploaderMetrics.java    From big-c with Apache License 2.0 4 votes vote down vote up
private SharedCacheUploaderMetrics() {
  registry = new MetricsRegistry("SharedCacheUploaderRequests");
  LOG.debug("Initialized "+ registry);
}
 
Example #28
Source File: TestMetricsSourceAdapter.java    From hadoop with Apache License 2.0 4 votes vote down vote up
TestSource(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #29
Source File: TestGangliaMetrics.java    From hadoop with Apache License 2.0 4 votes vote down vote up
TestSource(String recName) {
  registry = new MetricsRegistry(recName);
}
 
Example #30
Source File: TestMetricsSystemImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
TestSource2(String recName) {
  registry = new MetricsRegistry(recName);
}