Java Code Examples for org.apache.hadoop.metrics2.MetricsSink

The following examples show how to use org.apache.hadoop.metrics2.MetricsSink. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: MetricsSystemImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override public synchronized <T extends MetricsSink>
T register(final String name, final String description, final T sink) {
  LOG.debug(name +", "+ description);
  if (allSinks.containsKey(name)) {
    LOG.warn("Sink "+ name +" already exists!");
    return sink;
  }
  allSinks.put(name, sink);
  if (config != null) {
    registerSink(name, description, sink);
  }
  // We want to re-register the sink to pick up new config
  // when the metrics system restarts.
  register(name, new AbstractCallback() {
    @Override public void postStart() {
      register(name, description, sink);
    }
  });
  return sink;
}
 
Example 2
Source Project: big-c   Source File: MetricsSystemImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override public synchronized <T extends MetricsSink>
T register(final String name, final String description, final T sink) {
  LOG.debug(name +", "+ description);
  if (allSinks.containsKey(name)) {
    LOG.warn("Sink "+ name +" already exists!");
    return sink;
  }
  allSinks.put(name, sink);
  if (config != null) {
    registerSink(name, description, sink);
  }
  // We want to re-register the sink to pick up new config
  // when the metrics system restarts.
  register(name, new AbstractCallback() {
    @Override public void postStart() {
      register(name, description, sink);
    }
  });
  return sink;
}
 
Example 3
Source Project: hadoop   Source File: MetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
synchronized void registerSink(String name, String desc, MetricsSink sink) {
  checkNotNull(config, "config");
  MetricsConfig conf = sinkConfigs.get(name);
  MetricsSinkAdapter sa = conf != null
      ? newSink(name, desc, sink, conf)
      : newSink(name, desc, sink, config.subset(SINK_KEY));
  sinks.put(name, sa);
  sa.start();
  LOG.info("Registered sink "+ name);
}
 
Example 4
Source Project: hadoop   Source File: MetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink,
                                  MetricsConfig conf) {
  return new MetricsSinkAdapter(name, desc, sink, conf.getString(CONTEXT_KEY),
      conf.getFilter(SOURCE_FILTER_KEY),
      conf.getFilter(RECORD_FILTER_KEY),
      conf.getFilter(METRIC_FILTER_KEY),
      conf.getInt(PERIOD_KEY, PERIOD_DEFAULT),
      conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT),
      conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT),
      conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT),
      conf.getInt(RETRY_COUNT_KEY, RETRY_COUNT_DEFAULT));
}
 
Example 5
Source Project: hadoop   Source File: MetricsSinkAdapter.java    License: Apache License 2.0 5 votes vote down vote up
MetricsSinkAdapter(String name, String description, MetricsSink sink,
                   String context, MetricsFilter sourceFilter,
                   MetricsFilter recordFilter, MetricsFilter metricFilter,
                   int period, int queueCapacity, int retryDelay,
                   float retryBackoff, int retryCount) {
  this.name = checkNotNull(name, "name");
  this.description = description;
  this.sink = checkNotNull(sink, "sink object");
  this.context = context;
  this.sourceFilter = sourceFilter;
  this.recordFilter = recordFilter;
  this.metricFilter = metricFilter;
  this.period = checkArg(period, period > 0, "period");
  firstRetryDelay = checkArg(retryDelay, retryDelay > 0, "retry delay");
  this.retryBackoff = checkArg(retryBackoff, retryBackoff>1, "retry backoff");
  oobPutTimeout = (long)
      (firstRetryDelay * Math.pow(retryBackoff, retryCount) * 1000);
  this.retryCount = retryCount;
  this.queue = new SinkQueue<MetricsBuffer>(checkArg(queueCapacity,
      queueCapacity > 0, "queue capacity"));
  latency = registry.newRate("Sink_"+ name, "Sink end to end latency", false);
  dropped = registry.newCounter("Sink_"+ name +"Dropped",
                                "Dropped updates per sink", 0);
  qsize = registry.newGauge("Sink_"+ name + "Qsize", "Queue size", 0);

  sinkThread = new Thread() {
    @Override public void run() {
      publishMetricsFromQueue();
    }
  };
  sinkThread.setName(name);
  sinkThread.setDaemon(true);
}
 
Example 6
Source Project: hadoop   Source File: TestMetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testInitFirstVerifyCallBacks() throws Exception {
  DefaultMetricsSystem.shutdown(); 
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics

  try {
    verify(sink1, timeout(200).times(2)).putMetrics(r1.capture());
    verify(sink2, timeout(200).times(2)).putMetrics(r2.capture());
  } finally {
    ms.stop();
    ms.shutdown();
  }
  //When we call stop, at most two sources will be consumed by each sink thread.
  List<MetricsRecord> mr1 = r1.getAllValues();
  List<MetricsRecord> mr2 = r2.getAllValues();
  checkMetricsRecords(mr1);
  assertEquals("output", mr1, mr2);

}
 
Example 7
Source Project: big-c   Source File: MetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
synchronized void registerSink(String name, String desc, MetricsSink sink) {
  checkNotNull(config, "config");
  MetricsConfig conf = sinkConfigs.get(name);
  MetricsSinkAdapter sa = conf != null
      ? newSink(name, desc, sink, conf)
      : newSink(name, desc, sink, config.subset(SINK_KEY));
  sinks.put(name, sa);
  sa.start();
  LOG.info("Registered sink "+ name);
}
 
Example 8
Source Project: big-c   Source File: MetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink,
                                  MetricsConfig conf) {
  return new MetricsSinkAdapter(name, desc, sink, conf.getString(CONTEXT_KEY),
      conf.getFilter(SOURCE_FILTER_KEY),
      conf.getFilter(RECORD_FILTER_KEY),
      conf.getFilter(METRIC_FILTER_KEY),
      conf.getInt(PERIOD_KEY, PERIOD_DEFAULT),
      conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT),
      conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT),
      conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT),
      conf.getInt(RETRY_COUNT_KEY, RETRY_COUNT_DEFAULT));
}
 
Example 9
Source Project: big-c   Source File: MetricsSinkAdapter.java    License: Apache License 2.0 5 votes vote down vote up
MetricsSinkAdapter(String name, String description, MetricsSink sink,
                   String context, MetricsFilter sourceFilter,
                   MetricsFilter recordFilter, MetricsFilter metricFilter,
                   int period, int queueCapacity, int retryDelay,
                   float retryBackoff, int retryCount) {
  this.name = checkNotNull(name, "name");
  this.description = description;
  this.sink = checkNotNull(sink, "sink object");
  this.context = context;
  this.sourceFilter = sourceFilter;
  this.recordFilter = recordFilter;
  this.metricFilter = metricFilter;
  this.period = checkArg(period, period > 0, "period");
  firstRetryDelay = checkArg(retryDelay, retryDelay > 0, "retry delay");
  this.retryBackoff = checkArg(retryBackoff, retryBackoff>1, "retry backoff");
  oobPutTimeout = (long)
      (firstRetryDelay * Math.pow(retryBackoff, retryCount) * 1000);
  this.retryCount = retryCount;
  this.queue = new SinkQueue<MetricsBuffer>(checkArg(queueCapacity,
      queueCapacity > 0, "queue capacity"));
  latency = registry.newRate("Sink_"+ name, "Sink end to end latency", false);
  dropped = registry.newCounter("Sink_"+ name +"Dropped",
                                "Dropped updates per sink", 0);
  qsize = registry.newGauge("Sink_"+ name + "Qsize", "Queue size", 0);

  sinkThread = new Thread() {
    @Override public void run() {
      publishMetricsFromQueue();
    }
  };
  sinkThread.setName(name);
  sinkThread.setDaemon(true);
}
 
Example 10
Source Project: big-c   Source File: TestMetricsSystemImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testInitFirstVerifyCallBacks() throws Exception {
  DefaultMetricsSystem.shutdown(); 
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics

  try {
    verify(sink1, timeout(200).times(2)).putMetrics(r1.capture());
    verify(sink2, timeout(200).times(2)).putMetrics(r2.capture());
  } finally {
    ms.stop();
    ms.shutdown();
  }
  //When we call stop, at most two sources will be consumed by each sink thread.
  List<MetricsRecord> mr1 = r1.getAllValues();
  List<MetricsRecord> mr2 = r2.getAllValues();
  checkMetricsRecords(mr1);
  assertEquals("output", mr1, mr2);

}
 
Example 11
Source Project: hadoop   Source File: MetricsSystemImpl.java    License: Apache License 2.0 4 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc,
                                  MetricsConfig conf) {
  return newSink(name, desc, (MetricsSink) conf.getPlugin(""), conf);
}
 
Example 12
Source Project: hadoop   Source File: MetricsSinkAdapter.java    License: Apache License 2.0 4 votes vote down vote up
MetricsSink sink() {
  return sink;
}
 
Example 13
Source Project: hadoop   Source File: TestMetricsSystemImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
  DefaultMetricsSystem.shutdown();
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics
  ms.stop();
  ms.shutdown();

  //When we call stop, at most two sources will be consumed by each sink thread.
  verify(sink1, atMost(2)).putMetrics(r1.capture());
  List<MetricsRecord> mr1 = r1.getAllValues();
  verify(sink2, atMost(2)).putMetrics(r2.capture());
  List<MetricsRecord> mr2 = r2.getAllValues();
  if (mr1.size() != 0 && mr2.size() != 0) {
    checkMetricsRecords(mr1);
    assertEquals("output", mr1, mr2);
  } else if (mr1.size() != 0) {
    checkMetricsRecords(mr1);
  } else if (mr2.size() != 0) {
    checkMetricsRecords(mr2);
  }
}
 
Example 14
Source Project: big-c   Source File: MetricsSystemImpl.java    License: Apache License 2.0 4 votes vote down vote up
static MetricsSinkAdapter newSink(String name, String desc,
                                  MetricsConfig conf) {
  return newSink(name, desc, (MetricsSink) conf.getPlugin(""), conf);
}
 
Example 15
Source Project: big-c   Source File: MetricsSinkAdapter.java    License: Apache License 2.0 4 votes vote down vote up
MetricsSink sink() {
  return sink;
}
 
Example 16
Source Project: big-c   Source File: TestMetricsSystemImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
  DefaultMetricsSystem.shutdown();
  new ConfigBuilder().add("*.period", 8)
      //.add("test.sink.plugin.urls", getPluginUrlsAsString())
      .add("test.sink.test.class", TestSink.class.getName())
      .add("test.*.source.filter.exclude", "s0")
      .add("test.source.s1.metric.filter.exclude", "X*")
      .add("test.sink.sink1.metric.filter.exclude", "Y*")
      .add("test.sink.sink2.metric.filter.exclude", "Y*")
      .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
  MetricsSystemImpl ms = new MetricsSystemImpl("Test");
  ms.start();
  ms.register("s0", "s0 desc", new TestSource("s0rec"));
  TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
  s1.c1.incr();
  s1.xxx.incr();
  s1.g1.set(2);
  s1.yyy.incr(2);
  s1.s1.add(0);
  MetricsSink sink1 = mock(MetricsSink.class);
  MetricsSink sink2 = mock(MetricsSink.class);
  ms.registerSink("sink1", "sink1 desc", sink1);
  ms.registerSink("sink2", "sink2 desc", sink2);
  ms.publishMetricsNow(); // publish the metrics
  ms.stop();
  ms.shutdown();

  //When we call stop, at most two sources will be consumed by each sink thread.
  verify(sink1, atMost(2)).putMetrics(r1.capture());
  List<MetricsRecord> mr1 = r1.getAllValues();
  verify(sink2, atMost(2)).putMetrics(r2.capture());
  List<MetricsRecord> mr2 = r2.getAllValues();
  if (mr1.size() != 0 && mr2.size() != 0) {
    checkMetricsRecords(mr1);
    assertEquals("output", mr1, mr2);
  } else if (mr1.size() != 0) {
    checkMetricsRecords(mr1);
  } else if (mr2.size() != 0) {
    checkMetricsRecords(mr2);
  }
}
 
Example 17
Source Project: phoenix   Source File: TracingTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void registerSink(MetricsSink sink){
    Metrics.initialize().register("phoenix", "test sink gets logged", sink);
}
 
Example 18
Source Project: phoenix   Source File: TracingTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void registerSink(MetricsSink sink, String name){
    Metrics.initialize().register(name, "test sink gets logged", sink);
}