org.apache.hadoop.metrics2.MetricsSink Java Examples

The following examples show how to use org.apache.hadoop.metrics2.MetricsSink. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetricsSystemImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override public synchronized <T extends MetricsSink>
T register(final String name, final String description, final T sink) {
  LOG.debug(name +", "+ description);
  if (allSinks.containsKey(name)) {
    LOG.warn("Sink "+ name +" already exists!");
    return sink;
  }
  allSinks.put(name, sink);
  if (config != null) {
    registerSink(name, description, sink);
  }
  // We want to re-register the sink to pick up new config
  // when the metrics system restarts.
  register(name, new AbstractCallback() {
    @Override public void postStart() {
      register(name, description, sink);
    }
  });
  return sink;
}
 
Example #2
Source File: MetricsSystemImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override public synchronized <T extends MetricsSink>
T register(final String name, final String description, final T sink) {
  LOG.debug(name +", "+ description);
  if (allSinks.containsKey(name)) {
    LOG.warn("Sink "+ name +" already exists!");
    return sink;
  }
  allSinks.put(name, sink);
  if (config != null) {
    registerSink(name, description, sink);
  }
  // We want to re-register the sink to pick up new config
  // when the metrics system restarts.
  register(name, new AbstractCallback() {
    @Override public void postStart() {
      register(name, description, sink);
    }
  });
  return sink;
}
 
Example #3
Source File: MetricsSystemImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
synchronized void registerSink(String name, String desc, MetricsSink sink) {
  checkNotNull(config, "config");
  MetricsConfig conf = sinkConfigs.get(name);
  MetricsSinkAdapter sa = conf != null
      ? newSink(name, desc, sink, conf)
      : newSink(name, desc, sink, config.subset(SINK_KEY));
  sinks.put(name, sa);
  sa.start();
  LOG.info("Registered sink "+ name);
}
 
Example #4
Source File: TestMetricsSystemImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test public void testInitFirstVerifyCallBacks() throws Exception {
  DefaultMetricsSystem.shutdown(); 
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics

  try {
    verify(sink1, timeout(200).times(2)).putMetrics(r1.capture());
    verify(sink2, timeout(200).times(2)).putMetrics(r2.capture());
  } finally {
    ms.stop();
    ms.shutdown();
  }
  //When we call stop, at most two sources will be consumed by each sink thread.
  List<MetricsRecord> mr1 = r1.getAllValues();
  List<MetricsRecord> mr2 = r2.getAllValues();
  checkMetricsRecords(mr1);
  assertEquals("output", mr1, mr2);

}
 
Example #5
Source File: MetricsSinkAdapter.java    From big-c with Apache License 2.0 5 votes vote down vote up
MetricsSinkAdapter(String name, String description, MetricsSink sink,
                   String context, MetricsFilter sourceFilter,
                   MetricsFilter recordFilter, MetricsFilter metricFilter,
                   int period, int queueCapacity, int retryDelay,
                   float retryBackoff, int retryCount) {
  this.name = checkNotNull(name, "name");
  this.description = description;
  this.sink = checkNotNull(sink, "sink object");
  this.context = context;
  this.sourceFilter = sourceFilter;
  this.recordFilter = recordFilter;
  this.metricFilter = metricFilter;
  this.period = checkArg(period, period > 0, "period");
  firstRetryDelay = checkArg(retryDelay, retryDelay > 0, "retry delay");
  this.retryBackoff = checkArg(retryBackoff, retryBackoff>1, "retry backoff");
  oobPutTimeout = (long)
      (firstRetryDelay * Math.pow(retryBackoff, retryCount) * 1000);
  this.retryCount = retryCount;
  this.queue = new SinkQueue<MetricsBuffer>(checkArg(queueCapacity,
      queueCapacity > 0, "queue capacity"));
  latency = registry.newRate("Sink_"+ name, "Sink end to end latency", false);
  dropped = registry.newCounter("Sink_"+ name +"Dropped",
                                "Dropped updates per sink", 0);
  qsize = registry.newGauge("Sink_"+ name + "Qsize", "Queue size", 0);

  sinkThread = new Thread() {
    @Override public void run() {
      publishMetricsFromQueue();
    }
  };
  sinkThread.setName(name);
  sinkThread.setDaemon(true);
}
 
Example #6
Source File: MetricsSystemImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink,
                                  MetricsConfig conf) {
  return new MetricsSinkAdapter(name, desc, sink, conf.getString(CONTEXT_KEY),
      conf.getFilter(SOURCE_FILTER_KEY),
      conf.getFilter(RECORD_FILTER_KEY),
      conf.getFilter(METRIC_FILTER_KEY),
      conf.getInt(PERIOD_KEY, PERIOD_DEFAULT),
      conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT),
      conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT),
      conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT),
      conf.getInt(RETRY_COUNT_KEY, RETRY_COUNT_DEFAULT));
}
 
Example #7
Source File: MetricsSystemImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
synchronized void registerSink(String name, String desc, MetricsSink sink) {
  checkNotNull(config, "config");
  MetricsConfig conf = sinkConfigs.get(name);
  MetricsSinkAdapter sa = conf != null
      ? newSink(name, desc, sink, conf)
      : newSink(name, desc, sink, config.subset(SINK_KEY));
  sinks.put(name, sa);
  sa.start();
  LOG.info("Registered sink "+ name);
}
 
Example #8
Source File: TestMetricsSystemImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test public void testInitFirstVerifyCallBacks() throws Exception {
  DefaultMetricsSystem.shutdown(); 
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics

  try {
    verify(sink1, timeout(200).times(2)).putMetrics(r1.capture());
    verify(sink2, timeout(200).times(2)).putMetrics(r2.capture());
  } finally {
    ms.stop();
    ms.shutdown();
  }
  //When we call stop, at most two sources will be consumed by each sink thread.
  List<MetricsRecord> mr1 = r1.getAllValues();
  List<MetricsRecord> mr2 = r2.getAllValues();
  checkMetricsRecords(mr1);
  assertEquals("output", mr1, mr2);

}
 
Example #9
Source File: MetricsSinkAdapter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
MetricsSinkAdapter(String name, String description, MetricsSink sink,
                   String context, MetricsFilter sourceFilter,
                   MetricsFilter recordFilter, MetricsFilter metricFilter,
                   int period, int queueCapacity, int retryDelay,
                   float retryBackoff, int retryCount) {
  this.name = checkNotNull(name, "name");
  this.description = description;
  this.sink = checkNotNull(sink, "sink object");
  this.context = context;
  this.sourceFilter = sourceFilter;
  this.recordFilter = recordFilter;
  this.metricFilter = metricFilter;
  this.period = checkArg(period, period > 0, "period");
  firstRetryDelay = checkArg(retryDelay, retryDelay > 0, "retry delay");
  this.retryBackoff = checkArg(retryBackoff, retryBackoff>1, "retry backoff");
  oobPutTimeout = (long)
      (firstRetryDelay * Math.pow(retryBackoff, retryCount) * 1000);
  this.retryCount = retryCount;
  this.queue = new SinkQueue<MetricsBuffer>(checkArg(queueCapacity,
      queueCapacity > 0, "queue capacity"));
  latency = registry.newRate("Sink_"+ name, "Sink end to end latency", false);
  dropped = registry.newCounter("Sink_"+ name +"Dropped",
                                "Dropped updates per sink", 0);
  qsize = registry.newGauge("Sink_"+ name + "Qsize", "Queue size", 0);

  sinkThread = new Thread() {
    @Override public void run() {
      publishMetricsFromQueue();
    }
  };
  sinkThread.setName(name);
  sinkThread.setDaemon(true);
}
 
Example #10
Source File: MetricsSystemImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink,
                                  MetricsConfig conf) {
  return new MetricsSinkAdapter(name, desc, sink, conf.getString(CONTEXT_KEY),
      conf.getFilter(SOURCE_FILTER_KEY),
      conf.getFilter(RECORD_FILTER_KEY),
      conf.getFilter(METRIC_FILTER_KEY),
      conf.getInt(PERIOD_KEY, PERIOD_DEFAULT),
      conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT),
      conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT),
      conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT),
      conf.getInt(RETRY_COUNT_KEY, RETRY_COUNT_DEFAULT));
}
 
Example #11
Source File: TestMetricsSystemImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
  DefaultMetricsSystem.shutdown();
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics
  ms.stop();
  ms.shutdown();

  //When we call stop, at most two sources will be consumed by each sink thread.
  verify(sink1, atMost(2)).putMetrics(r1.capture());
  List<MetricsRecord> mr1 = r1.getAllValues();
  verify(sink2, atMost(2)).putMetrics(r2.capture());
  List<MetricsRecord> mr2 = r2.getAllValues();
  if (mr1.size() != 0 && mr2.size() != 0) {
    checkMetricsRecords(mr1);
    assertEquals("output", mr1, mr2);
  } else if (mr1.size() != 0) {
    checkMetricsRecords(mr1);
  } else if (mr2.size() != 0) {
    checkMetricsRecords(mr2);
  }
}
 
Example #12
Source File: MetricsSinkAdapter.java    From hadoop with Apache License 2.0 4 votes vote down vote up
MetricsSink sink() {
  return sink;
}
 
Example #13
Source File: MetricsSystemImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc,
                                  MetricsConfig conf) {
  return newSink(name, desc, (MetricsSink) conf.getPlugin(""), conf);
}
 
Example #14
Source File: MetricsSystemImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc,
                                  MetricsConfig conf) {
  return newSink(name, desc, (MetricsSink) conf.getPlugin(""), conf);
}
 
Example #15
Source File: MetricsSinkAdapter.java    From big-c with Apache License 2.0 4 votes vote down vote up
MetricsSink sink() {
  return sink;
}
 
Example #16
Source File: TestMetricsSystemImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
  DefaultMetricsSystem.shutdown();
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics
  ms.stop();
  ms.shutdown();

  //When we call stop, at most two sources will be consumed by each sink thread.
  verify(sink1, atMost(2)).putMetrics(r1.capture());
  List<MetricsRecord> mr1 = r1.getAllValues();
  verify(sink2, atMost(2)).putMetrics(r2.capture());
  List<MetricsRecord> mr2 = r2.getAllValues();
  if (mr1.size() != 0 && mr2.size() != 0) {
    checkMetricsRecords(mr1);
    assertEquals("output", mr1, mr2);
  } else if (mr1.size() != 0) {
    checkMetricsRecords(mr1);
  } else if (mr2.size() != 0) {
    checkMetricsRecords(mr2);
  }
}
 
Example #17
Source File: TracingTestUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static void registerSink(MetricsSink sink){
    Metrics.initialize().register("phoenix", "test sink gets logged", sink);
}
 
Example #18
Source File: TracingTestUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static void registerSink(MetricsSink sink, String name){
    Metrics.initialize().register(name, "test sink gets logged", sink);
}