com.yammer.metrics.core.Gauge Java Examples

The following examples show how to use com.yammer.metrics.core.Gauge. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: ambari-metrics   Author: apache   File: ScheduledReporterTest.java    License: Apache License 2.0 6 votes vote down vote up
private Set<Entry<MetricName, Metric>> set(List<Metric> metrics) {
  final Map<MetricName, Metric> map = new HashMap<MetricName, Metric>();
  for (Metric metric : metrics) {
    String name = null;
    if (metric instanceof Gauge) {
      name = "gauge";
    } else if (metric instanceof Counter) {
      name = "counter";
    } else if (metric instanceof Histogram) {
      name = "histogram";
    } else if (metric instanceof Meter) {
      name = "meter";
    } else if (metric instanceof Timer) {
      name = "timer";
    }
    map.put(new MetricName(System.class, name), metric);
  }
  return map.entrySet();
}
 
Example #2
Source Project: kafka-metrics   Author: amient   File: GroupMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public T get(String group, TopicPartition tp) {
    Map<TopicPartition, T> metrics = data.get(group);
    if (metrics == null) {
        metrics = new HashMap<>();
        data.put(group, metrics);
    }
    T metric = metrics.get(tp);
    if (metric == null) {
        try {
            metric = cls.newInstance();
            if (metric instanceof Gauge) {
                registry.newGauge(NewName(group, tp), (Gauge)metric);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        metrics.put(tp, metric);
    }
    return metric;
}
 
Example #3
Source Project: kafka-graphite   Author: damienclaveau   File: FilterMetricPredicateTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void keepGaugesIfTheyThrowRuntimeExceptions() throws Exception {
    MetricPredicate predicate = new FilterMetricPredicate();

    MetricName metricName = new MetricName("test", "test", "delete", "scope", "mBeanName");

    Metric gauge = Metrics.newGauge(metricName, new Gauge<Long>() {
        @Override
        public Long value() {
            throw new RuntimeException("catch me if you can");
        }
    });

    assertTrue(predicate.matches(metricName, gauge));

    assertTrue("The gauge should be there", Metrics.defaultRegistry().allMetrics().containsKey(metricName));
    assertEquals(Metrics.defaultRegistry().allMetrics().get(metricName), gauge);
}
 
Example #4
Source Project: stratio-cassandra   Author: Stratio   File: ClientMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public void addCounter(String name, final Callable<Integer> provider)
{
    Metrics.newGauge(factory.createMetricName(name), new Gauge<Integer>()
    {
        public Integer value()
        {
            try
            {
                return provider.call();
            }
            catch (Exception e)
            {
                throw new RuntimeException(e);
            }
        }
    });
}
 
Example #5
Source Project: stratio-cassandra   Author: Stratio   File: CommitLogMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public CommitLogMetrics(final AbstractCommitLogService service, final CommitLogSegmentManager allocator)
{
    completedTasks = Metrics.newGauge(factory.createMetricName("CompletedTasks"), new Gauge<Long>()
    {
        public Long value()
        {
            return service.getCompletedTasks();
        }
    });
    pendingTasks = Metrics.newGauge(factory.createMetricName("PendingTasks"), new Gauge<Long>()
    {
        public Long value()
        {
            return service.getPendingTasks();
        }
    });
    totalCommitLogSize = Metrics.newGauge(factory.createMetricName("TotalCommitLogSize"), new Gauge<Long>()
    {
        public Long value()
        {
            return allocator.bytesUsed();
        }
    });
    waitingOnSegmentAllocation = Metrics.newTimer(factory.createMetricName("WaitingOnSegmentAllocation"), TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
    waitingOnCommit = Metrics.newTimer(factory.createMetricName("WaitingOnCommit"), TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
}
 
Example #6
Source Project: stratio-cassandra   Author: Stratio   File: FileCacheMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public FileCacheMetrics()
{
    hits = Metrics.newMeter(factory.createMetricName("Hits"), "hits", TimeUnit.SECONDS);
    requests = Metrics.newMeter(factory.createMetricName("Requests"), "requests", TimeUnit.SECONDS);
    hitRate = Metrics.newGauge(factory.createMetricName("HitRate"), new RatioGauge()
    {
        protected double getNumerator()
        {
            return hits.count();
        }

        protected double getDenominator()
        {
            return requests.count();
        }
    });
    size = Metrics.newGauge(factory.createMetricName("Size"), new Gauge<Long>()
    {
        public Long value()
        {
            return FileCacheService.instance.sizeInBytes();
        }
    });
}
 
Example #7
Source Project: stratio-cassandra   Author: Stratio   File: CQLMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public CQLMetrics()
{
    regularStatementsExecuted = Metrics.newCounter(factory.createMetricName("RegularStatementsExecuted"));
    preparedStatementsExecuted = Metrics.newCounter(factory.createMetricName("PreparedStatementsExecuted"));
    preparedStatementsEvicted = Metrics.newCounter(factory.createMetricName("PreparedStatementsEvicted"));

    preparedStatementsCount = Metrics.newGauge(factory.createMetricName("PreparedStatementsCount"), new Gauge<Integer>()
    {
        public Integer value()
        {
            return QueryProcessor.preparedStatementsCount();
        }
    });
    preparedStatementsRatio = Metrics.newGauge(factory.createMetricName("PreparedStatementsRatio"), new RatioGauge()
    {
        public double getNumerator()
        {
            return preparedStatementsExecuted.count();
        }

        public double getDenominator()
        {
            return regularStatementsExecuted.count() + preparedStatementsExecuted.count();
        }
    });
}
 
Example #8
Source Project: incubator-retired-blur   Author: apache   File: DeepPagingCache.java    License: Apache License 2.0 6 votes vote down vote up
public DeepPagingCache(long maxEntriesForDeepPaging) {
  _hits = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, DEEP_PAGING_CACHE, HIT), HIT, TimeUnit.SECONDS);
  _misses = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, DEEP_PAGING_CACHE, MISS), MISS, TimeUnit.SECONDS);
  _evictions = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, DEEP_PAGING_CACHE, EVICTION), EVICTION,
      TimeUnit.SECONDS);
  _lruCache = new ConcurrentLinkedHashMap.Builder<DeepPageKeyPlusPosition, DeepPageContainer>()
      .maximumWeightedCapacity(maxEntriesForDeepPaging)
      .listener(new EvictionListener<DeepPageKeyPlusPosition, DeepPageContainer>() {
        @Override
        public void onEviction(DeepPageKeyPlusPosition key, DeepPageContainer value) {
          _positionCache.remove(key);
          _evictions.mark();
        }
      }).build();
  Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, DEEP_PAGING_CACHE, SIZE), new Gauge<Long>() {
    @Override
    public Long value() {
      return _lruCache.weightedSize();
    }
  });
  _positionCache = new ConcurrentSkipListMap<DeepPageKeyPlusPosition, DeepPageContainer>();
}
 
Example #9
Source Project: signalfx-java   Author: signalfx   File: YammerExample.java    License: Apache License 2.0 6 votes vote down vote up
private static Metric getCumulativeCounter(MetricsRegistry metricsRegistry,
                                           MetricMetadata metricMetadata) {
    MetricName counterCallbackName = new MetricName(YammerExample.class, "yammer.test.cumulativeCounter");
    Metric cumulativeCounter = SfUtil.cumulativeCounter(
            metricsRegistry,
            counterCallbackName,
            metricMetadata,
            new Gauge<Long>() {

                private long i = 0;

                @Override
                public Long value() {
                    return i++;
                }

            });

    metricMetadata.forMetric(cumulativeCounter)
            .withSourceName(SIGNAL_FX)
            .withDimension(LIBRARY_VERSION, YAMMER);

    return cumulativeCounter;
}
 
Example #10
Source Project: ambari-metrics   Author: apache   File: KafkaTimelineMetricsReporter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void processGauge(MetricName name, Gauge<?> gauge, Context context) throws Exception {
  final long currentTimeMillis = System.currentTimeMillis();
  final String sanitizedName = sanitizeName(name);

  try {
    if (!isExcludedMetric(sanitizedName)) {
      cacheSanitizedTimelineMetric(currentTimeMillis, sanitizedName, "", Double.parseDouble(String.valueOf(gauge.value())));
      populateMetricsList(context, MetricType.GAUGE, sanitizedName);
    }
  } catch (NumberFormatException ex) {
    LOG.debug(ex.getMessage());
  }
}
 
Example #11
Source Project: ambari-metrics   Author: apache   File: KafkaTimelineMetricsReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  @SuppressWarnings({ "rawtypes", "unchecked" })
  Gauge g = registry.newGauge(System.class, "gauge", gauge);
  Counter counter = registry.newCounter(System.class, "counter");
  Histogram histogram = registry.newHistogram(System.class, "histogram");
  Meter meter = registry.newMeter(System.class, "meter", "empty", TimeUnit.MILLISECONDS);
  Timer timer = registry.newTimer(System.class, "timer");
  list.add(g);
  list.add(counter);
  list.add(histogram);
  list.add(meter);
  list.add(timer);
  Properties properties = new Properties();
  properties.setProperty("zookeeper.connect", "localhost:2181");
  properties.setProperty("kafka.timeline.metrics.sendInterval", "5900");
  properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000");
  properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188");
  properties.setProperty("kafka.timeline.metrics.port", "6188");
  properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true");
  properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c");
  properties.setProperty("external.kafka.metrics.include.prefix", "a.b.c.d");
  properties.setProperty("external.kafka.metrics.include.regex", "a.b.c.*.f");
  properties.setProperty("kafka.timeline.metrics.instanceId", "cluster");
  properties.setProperty("kafka.timeline.metrics.set.instanceId", "false");
  props = new VerifiableProperties(properties);
}
 
Example #12
Source Project: ambari-metrics   Author: apache   File: ScheduledReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void setUp() throws Exception {
  Gauge g = registry.newGauge(System.class, "gauge", gauge);
  Counter counter = registry.newCounter(System.class, "counter");
  Histogram histogram = registry.newHistogram(System.class, "histogram");
  Meter meter = registry.newMeter(System.class, "meter", "empty", TimeUnit.MILLISECONDS);
  Timer timer = registry.newTimer(System.class, "timer");
  list.add(g);
  list.add(counter);
  list.add(histogram);
  list.add(meter);
  list.add(timer);
  reporter.start(200, TimeUnit.MILLISECONDS);
}
 
Example #13
Source Project: cruise-control   Author: linkedin   File: YammerMetricProcessor.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public void processGauge(MetricName metricName, Gauge<?> gauge, Context context) {
  if (MetricsUtils.isInterested(metricName)) {
    LOG.trace("Processing metric {} of type Gauge.", metricName);
    if (!(gauge.value() instanceof Number)) {
      throw new IllegalStateException(String.format("The value of yammer metric %s is %s, which is not a number",
                                                    metricName, gauge.value()));
    }
    CruiseControlMetric ccm = MetricsUtils.toCruiseControlMetric(context.time(),
                                                                 context.brokerId(),
                                                                 metricName,
                                                                 ((Number) gauge.value()).doubleValue());
    context.reporter().sendCruiseControlMetric(ccm);
  }
}
 
Example #14
Source Project: kafka-graphite   Author: damienclaveau   File: ValidatingMetricProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void processGauge(MetricName name, Gauge gauge, Object context) throws Exception {
    try {
        gauge.value();
    } catch (NoSuchElementException ex) {
        throw new InvalidGaugeException(String.format("%s.%s.%s", name.getGroup(), name.getType(), name.getName()), ex);
    }
}
 
Example #15
Source Project: kafka-graphite   Author: damienclaveau   File: FilterMetricPredicateTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void deleteGaugesIfTheyThrowNoSuchElementException() throws Exception {
    MetricPredicate predicate = new FilterMetricPredicate();

    MetricName metricNameToBeDeleted = new MetricName("test", "test", "delete", "scope", "mBeanName");

    Metric gaugeToBeDeleted = Metrics.newGauge(metricNameToBeDeleted, new Gauge<Long>() {
        @Override
        public Long value() {
            throw new NoSuchElementException("catch me if you can - i'm the the same as in KAFKA-1866");
        }
    });

    MetricName metricNameToStay = new MetricName("stay", "stay", "stay", "scope", "stay:mBeanName");
    Metric gaugeToStay = Metrics.newGauge(metricNameToStay, new Gauge<Long>() {
        @Override
        public Long value() {
            return 42L;
        }
    });


    assertFalse(predicate.matches(metricNameToBeDeleted, gaugeToBeDeleted));
    assertTrue(predicate.matches(metricNameToStay, gaugeToStay));


    assertFalse("The gauge should be deleted", Metrics.defaultRegistry().allMetrics().containsKey(metricNameToBeDeleted));
    assertTrue("The gauge should be there", Metrics.defaultRegistry().allMetrics().containsKey(metricNameToStay));
    assertEquals(Metrics.defaultRegistry().allMetrics().get(metricNameToStay), gaugeToStay);
}
 
Example #16
Source Project: kafka-statsd-metrics2   Author: airbnb   File: StatsDReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void isTaggedTest() {
  registry.add(new MetricName("kafka.common", "AppInfo", "Version", null, "kafka.common:type=AppInfo,name=Version"),
      new Gauge<String>() {
        public String value() {
          return "0.8.2";
        }
      });
  assertTrue(((StatsDReporter) reporter).isTagged(registry.allMetrics()));
}
 
Example #17
Source Project: kafka-statsd-metrics2   Author: airbnb   File: StatsDReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public final void longGauge() throws Exception {
  final long value = 0xdeadbeef;
  addMetricAndRunReporter(
      new Callable<Gauge<Object>>() {
        @Override
        public Gauge<Object> call() throws Exception {
          return createGauge(value);
        }
      });
  verifySend(value);
}
 
Example #18
Source Project: kafka-statsd-metrics2   Author: airbnb   File: StatsDReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void stringGauge() throws Exception {
  final String value = "The Metric";
  addMetricAndRunReporter(
      new Callable<Gauge<Object>>() {
        @Override
        public Gauge<Object> call() throws Exception {
          return createGauge(value);
        }
      });
  verify(statsD, never()).gauge(Matchers.anyString(), Matchers.anyDouble());
}
 
Example #19
Source Project: kafka-statsd-metrics2   Author: airbnb   File: StatsDReporterTest.java    License: Apache License 2.0 5 votes vote down vote up
static Gauge<Object> createGauge(Object value) throws Exception {
  @SuppressWarnings("unchecked")
  final Gauge<Object> mock = mock(Gauge.class);
  when(mock.value()).thenReturn(value);
  return configureMatcher(mock, doAnswer(new MetricsProcessorAction() {
    @Override
    void delegateToProcessor(MetricProcessor<Object> processor, MetricName name, Object context) throws Exception {
      processor.processGauge(name, mock, context);
    }
  }));
}
 
Example #20
Source Project: incubator-retired-blur   Author: apache   File: SharedMergeScheduler.java    License: Apache License 2.0 5 votes vote down vote up
public SharedMergeScheduler(int threads, long smallMergeThreshold) {
  MetricName mergeSmallQueueDepth = new MetricName(ORG_APACHE_BLUR, LUCENE, SMALL_QUEUE_DEPTH);
  MetricName mergeSmallQueueDepthInBytes = new MetricName(ORG_APACHE_BLUR, LUCENE, SMALL_QUEUE_DEPTH_IN_BYTES);
  MetricName mergeLargeQueueDepth = new MetricName(ORG_APACHE_BLUR, LUCENE, LARGE_QUEUE_DEPTH);
  MetricName mergeLargeQueueDepthInBytes = new MetricName(ORG_APACHE_BLUR, LUCENE, LARGE_QUEUE_DEPTH_IN_BYTES);

  _smallMergeThreshold = smallMergeThreshold;
  _smallMergeService = Executors.newThreadPool(SHARED_MERGE_SCHEDULER_PREFIX + "-small", threads, false);
  _largeMergeService = Executors.newThreadPool(SHARED_MERGE_SCHEDULER_PREFIX + "-large", threads, false);
  for (int i = 0; i < threads; i++) {
    _smallMergeService.submit(getMergerRunnable(_smallMergeQueue));
    _largeMergeService.submit(getMergerRunnable(_largeMergeQueue));
  }

  Metrics.newGauge(mergeSmallQueueDepth, new Gauge<Long>() {
    @Override
    public Long value() {
      return (long) _smallMergeQueue.size();
    }
  });
  Metrics.newGauge(mergeSmallQueueDepthInBytes, new Gauge<Long>() {
    @Override
    public Long value() {
      return getSizeInBytes(_smallMergeQueue);
    }
  });
  Metrics.newGauge(mergeLargeQueueDepth, new Gauge<Long>() {
    @Override
    public Long value() {
      return (long) _largeMergeQueue.size();
    }
  });
  Metrics.newGauge(mergeLargeQueueDepthInBytes, new Gauge<Long>() {
    @Override
    public Long value() {
      return getSizeInBytes(_largeMergeQueue);
    }
  });
}
 
Example #21
Source Project: incubator-retired-blur   Author: apache   File: ShardServerEventHandler.java    License: Apache License 2.0 5 votes vote down vote up
public ShardServerEventHandler() {
  Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections"), new Gauge<Long>() {
    @Override
    public Long value() {
      return null;
    }
  });
  _connectionMeter = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections/s"), "Connections/s",
      TimeUnit.SECONDS);
}
 
Example #22
Source Project: incubator-retired-blur   Author: apache   File: ControllerServerEventHandler.java    License: Apache License 2.0 5 votes vote down vote up
public ControllerServerEventHandler() {
  Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections"), new Gauge<Long>() {
    @Override
    public Long value() {
      return null;
    }
  });
  _connectionMeter = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections/s"), "Connections/s",
      TimeUnit.SECONDS);
}
 
Example #23
Source Project: incubator-retired-blur   Author: apache   File: ByteArrayPrimitiveFactory.java    License: Apache License 2.0 5 votes vote down vote up
public ByteArrayPrimitiveFactory(BlurConfiguration configuration) {
  super(configuration);
  Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, FST, SIZE), new Gauge<Long>() {
    @Override
    public Long value() {
      return _size.get();
    }
  });
}
 
Example #24
Source Project: incubator-retired-blur   Author: apache   File: JSONReporter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void processGauge(MetricName name, Gauge<?> gauge, Context context) throws Exception {
  MetricInfo info = context.getMetricInfo(name);
  long time = context.getTime();
  info.addNumber("timestamp", time);
  info.addNumber("value", getDouble(gauge.value()));
}
 
Example #25
Source Project: signalfx-java   Author: signalfx   File: YammerExample.java    License: Apache License 2.0 5 votes vote down vote up
private static Gauge getGauge(MetricsRegistry metricsRegistry, MetricMetadata metricMetadata) {
    Gauge gauge = metricsRegistry.newGauge(YammerExample.class, "yammer.test.gauge",
            new Gauge<Double>() {
                @Override
                public Double value() {
                    return Math.sin(System.currentTimeMillis() * 0.001 * 2 * Math.PI / 60);
                }
            });

    metricMetadata.forMetric(gauge)
            .withSourceName(SIGNAL_FX)
            .withDimension(LIBRARY_VERSION, YAMMER);
    return gauge;
}
 
Example #26
Source Project: incubator-pinot   Author: apache   File: AggregatedLongGauge.java    License: Apache License 2.0 5 votes vote down vote up
public void refresh() {
  long sum = 0;
  for (Gauge<T> gauge : _gauges) {
    sum += gauge.value().longValue();
  }
  _value = sum / _gauges.size();
}
 
Example #27
Source Project: netty-4.1.22   Author: tianheframe   File: CustomReporter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void processGauge(final MetricName name, final Gauge<?> gauge,
        final PrintStream stream) {
    stream.printf(locale, "    value = %s\n", gauge.value());
}
 
Example #28
Source Project: Halyard   Author: Merck   File: HalyardStats.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public int run(CommandLine cmd) throws Exception {
    String source = cmd.getOptionValue('s');
    String target = cmd.getOptionValue('t');
    String targetGraph = cmd.getOptionValue('g');
    String graphContext = cmd.getOptionValue('c');
    String thresh = cmd.getOptionValue('r');
    TableMapReduceUtil.addDependencyJars(getConf(),
        HalyardExport.class,
        NTriplesUtil.class,
        Rio.class,
        AbstractRDFHandler.class,
        RDFFormat.class,
        RDFParser.class,
        HTable.class,
        HBaseConfiguration.class,
        AuthenticationProtos.class,
        Trace.class,
        Gauge.class);
    HBaseConfiguration.addHbaseResources(getConf());
    Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target));
    job.getConfiguration().set(SOURCE, source);
    if (target != null) job.getConfiguration().set(TARGET, target);
    if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph);
    if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext);
    if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh));
    job.setJarByClass(HalyardStats.class);
    TableMapReduceUtil.initCredentials(job);

    Scan scan = HalyardTableUtils.scan(null, null);
    if (graphContext != null) { //restricting stats to scan given graph context only
        List<RowRange> ranges = new ArrayList<>();
        byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext));
        ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash));
        ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash));
        ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash));
        if (target == null) { //add stats context to the scanned row ranges (when in update mode) to delete the related stats during MapReduce
            ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph))));
        }
        scan.setFilter(new MultiRowRangeFilter(ranges));
    }
    TableMapReduceUtil.initTableMapperJob(
        source,
        scan,
        StatsMapper.class,
        ImmutableBytesWritable.class,
        LongWritable.class,
        job);
    job.setPartitionerClass(StatsPartitioner.class);
    job.setReducerClass(StatsReducer.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    if (job.waitForCompletion(true)) {
        LOG.info("Stats Generation Completed..");
        return 0;
    }
    return -1;
}
 
Example #29
Source Project: Halyard   Author: Merck   File: HalyardBulkExport.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected int run(CommandLine cmd) throws Exception {
    if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString());
    String source = cmd.getOptionValue('s');
    String queryFiles = cmd.getOptionValue('q');
    String target = cmd.getOptionValue('t');
    if (!target.contains("{0}")) {
        throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query.");
    }
    getConf().set(SOURCE, source);
    getConf().set(TARGET, target);
    String driver = cmd.getOptionValue('c');
    if (driver != null) {
        getConf().set(JDBC_DRIVER, driver);
    }
    String props[] = cmd.getOptionValues('p');
    if (props != null) {
        for (int i=0; i<props.length; i++) {
            props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8));
        }
        getConf().setStrings(JDBC_PROPERTIES, props);
    }
    if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i'));
    TableMapReduceUtil.addDependencyJars(getConf(),
           HalyardExport.class,
           NTriplesUtil.class,
           Rio.class,
           AbstractRDFHandler.class,
           RDFFormat.class,
           RDFParser.class,
           HTable.class,
           HBaseConfiguration.class,
           AuthenticationProtos.class,
           Trace.class,
           Gauge.class);
    HBaseConfiguration.addHbaseResources(getConf());
    String cp = cmd.getOptionValue('l');
    if (cp != null) {
        String jars[] = cp.split(":");
        StringBuilder newCp = new StringBuilder();
        for (int i=0; i<jars.length; i++) {
            if (i > 0) newCp.append(':');
            newCp.append(addTmpFile(jars[i])); //append clappspath entris to tmpfiles and trim paths from the classpath
        }
        getConf().set(JDBC_CLASSPATH, newCp.toString());
    }
    Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target);
    job.setJarByClass(HalyardBulkExport.class);
    job.setMaxMapAttempts(1);
    job.setMapperClass(BulkExportMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(Void.class);
    job.setNumReduceTasks(0);
    job.setInputFormatClass(QueryInputFormat.class);
    QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0);
    job.setOutputFormatClass(NullOutputFormat.class);
    TableMapReduceUtil.initCredentials(job);
    if (job.waitForCompletion(true)) {
        LOG.info("Bulk Export Completed..");
        return 0;
    }
    return -1;
}
 
Example #30
Source Project: Halyard   Author: Merck   File: HalyardBulkDelete.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public int run(CommandLine cmd) throws Exception {
    String source = cmd.getOptionValue('t');
    TableMapReduceUtil.addDependencyJars(getConf(),
        HalyardExport.class,
        NTriplesUtil.class,
        Rio.class,
        AbstractRDFHandler.class,
        RDFFormat.class,
        RDFParser.class,
        HTable.class,
        HBaseConfiguration.class,
        AuthenticationProtos.class,
        Trace.class,
        Gauge.class);
    HBaseConfiguration.addHbaseResources(getConf());
    Job job = Job.getInstance(getConf(), "HalyardDelete " + source);
    if (cmd.hasOption('s')) {
        job.getConfiguration().set(SUBJECT, cmd.getOptionValue('s'));
    }
    if (cmd.hasOption('p')) {
        job.getConfiguration().set(PREDICATE, cmd.getOptionValue('p'));
    }
    if (cmd.hasOption('o')) {
        job.getConfiguration().set(OBJECT, cmd.getOptionValue('o'));
    }
    if (cmd.hasOption('g')) {
        job.getConfiguration().setStrings(CONTEXTS, cmd.getOptionValues('g'));
    }
    job.setJarByClass(HalyardBulkDelete.class);
    TableMapReduceUtil.initCredentials(job);

    Scan scan = HalyardTableUtils.scan(null, null);

    TableMapReduceUtil.initTableMapperJob(source,
        scan,
        DeleteMapper.class,
        ImmutableBytesWritable.class,
        LongWritable.class,
        job);

    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(KeyValue.class);
    job.setSpeculativeExecution(false);
    job.setMapSpeculativeExecution(false);
    job.setReduceSpeculativeExecution(false);
    try (HTable hTable = HalyardTableUtils.getTable(getConf(), source, false, 0)) {
        HFileOutputFormat2.configureIncrementalLoad(job, hTable.getTableDescriptor(), hTable.getRegionLocator());
        FileOutputFormat.setOutputPath(job, new Path(cmd.getOptionValue('f')));
        TableMapReduceUtil.addDependencyJars(job);
        if (job.waitForCompletion(true)) {
            new LoadIncrementalHFiles(getConf()).doBulkLoad(new Path(cmd.getOptionValue('f')), hTable);
            LOG.info("Bulk Delete Completed..");
            return 0;
        }
    }
    return -1;
}