org.apache.beam.sdk.metrics.Metrics Java Examples

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KettleRow to KV<String,String> function", e );
    throw new RuntimeException( "Error in setup of KettleRow to KV<String,String> function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, counterName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch ( Exception e ) {
    errorCounter.inc();
    LOG.info( "Parse error on setup of Hop data to string lines : " + e.getMessage() );
    throw new RuntimeException( "Error on setup of converting Hop data to string lines", e );
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of HopRow to KV<String,String> function", e );
    throw new RuntimeException( "Error in setup of HopRow to KV<String,String> function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, counterName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init(transformPluginClasses, xpPluginClasses);
    groupRowMeta = JsonRowMeta.fromJson( groupRowMetaJson );
    subjectRowMeta = JsonRowMeta.fromJson( subjectRowMetaJson );
    aggregationTypes = new AggregationType[aggregations.length];
    for ( int i = 0; i < aggregationTypes.length; i++ ) {
      aggregationTypes[ i ] = AggregationType.getTypeFromName( aggregations[ i ] );
    }

    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of grouping by ", e);
    throw new RuntimeException( "Unable setup of group by ", e );
  }
}
 

@ProcessElement
public void processElement(ProcessContext c) {
  GenomicsFactory factory = GenomicsFactory.builder().build();
  Genomics genomics = factory.fromOfflineAuth(auth);

  processApiCall(genomics, c, c.element());

  Metrics.counter(GenomicsApiReader.class, "Genomics API Initialized Request Count")
      .inc(factory.initializedRequestsCount());
  Metrics.counter(GenomicsApiReader.class, "Genomics API Unsuccessful Response Count")
      .inc(factory.unsuccessfulResponsesCount());
  Metrics.counter(GenomicsApiReader.class, "Genomics API IOException Response Count")
      .inc(factory.ioExceptionsCount());
  LOG.info("ApiReader processed " + factory.initializedRequestsCount() + " requests ("
      + factory.unsuccessfulResponsesCount() + " server errors and "
      + factory.ioExceptionsCount() + " IO exceptions)");
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    Metrics.counter( Pipeline.METRIC_NAME_ERROR, transformName ).inc();
    LOG.error( "Error in setup of converting input data into Hop rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Hop rows", e );
  }
}
 

/** Generate Avro schema by reading one row. Expose Beam metrics via a Beam PTransform. */
public static Schema createSchema(
    final Pipeline pipeline, final JdbcExportArgs args, final Connection connection)
    throws Exception {
  final long startTime = System.nanoTime();
  final Schema generatedSchema = generateAvroSchema(args, connection);
  final long elapsedTimeSchema = (System.nanoTime() - startTime) / 1000000;
  LOGGER.info("Elapsed time to schema {} seconds", elapsedTimeSchema / 1000.0);

  final Counter cnt =
      Metrics.counter(BeamJdbcAvroSchema.class.getCanonicalName(), "schemaElapsedTimeMs");
  pipeline
      .apply(
          "ExposeSchemaCountersSeed",
          Create.of(Collections.singletonList(0)).withType(TypeDescriptors.integers()))
      .apply(
          "ExposeSchemaCounters",
          MapElements.into(TypeDescriptors.integers())
              .via(
                  v -> {
                    cnt.inc(elapsedTimeSchema);
                    return v;
                  }));
  return generatedSchema;
}
 

@ProcessElement
public void processElement(ProcessContext c) throws IOException, GeneralSecurityException, InterruptedException {
  Metrics.counter(RetrieveVariants.class, "Initialized Shard Count").inc();
  Stopwatch stopWatch = Stopwatch.createStarted();
  Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields);
  while (iter.hasNext()) {
    StreamVariantsResponse variantResponse = iter.next();
    c.output(variantResponse.getVariantsList());
  }
  stopWatch.stop();
  Metrics.distribution(RetrieveVariants.class, "Shard Processing Time (sec)")
      .update(stopWatch.elapsed(TimeUnit.SECONDS));
  Metrics.counter(RetrieveVariants.class, "Finished Shard Count").inc();
  stats.addValue(stopWatch.elapsed(TimeUnit.SECONDS));
  LOG.info("Shard Duration in Seconds - Min: " + stats.getMin() + " Max: " + stats.getMax() +
      " Avg: " + stats.getMean() + " StdDev: " + stats.getStandardDeviation());
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch(Exception e) {
    numErrors.inc();
    LOG.error( "Error in setup of String to Hop Row conversion function", e );
    throw new RuntimeException( "Error in setup of String to Hop Row conversion function", e );
  }
}
 

@VisibleForTesting
static Counter getOrCreateCounter(@Nullable Map<String, String> attributes, String name) {
  if (attributes == null) {
    attributes = EMPTY_ATTRIBUTES;
  }
  String namespace = attributes.getOrDefault("document_namespace", "unknown_namespace");
  // Dataflow's UI collapses the metric name, but always shows the penultimate path component
  // as part of "Step", so we're a bit more verbose with the default doctype value.
  String docType = attributes.getOrDefault("document_type", "unknown_doctype");
  String version = attributes.get("document_version");
  if (version != null) {
    docType = docType + "_v" + version;
  }
  String key = String.format("%s/%s/%s", namespace, docType, name)
      // We change dashes to underscores to make sure we're following Stackdriver's naming scheme:
      // https://cloud.google.com/monitoring/api/v3/metrics-details#label_names
      .replace("-", "_");
  return counters.computeIfAbsent(key, k -> Metrics.counter(PerDocTypeCounter.class, k));
}
 

@StartBundle
public void startBundle(DoFn<Read, String>.StartBundleContext c) throws IOException {
  LOG.info("Starting bundle ");
  storage = Transport.newStorageClient(c.getPipelineOptions().as(GCSOptions.class)).build().objects();

  Metrics.counter(WriteBAMFn.class, "Initialized Write Shard Count").inc();
  stopWatch = Stopwatch.createStarted();

  options = c.getPipelineOptions().as(Options.class);

  readCount = 0;
  unmappedReadCount = 0;
  headerInfo = null;
  prevRead = null;
  minAlignment = Long.MAX_VALUE;
  maxAlignment = Long.MIN_VALUE;
  hadOutOfOrder = false;
}
 

@FinishBundle
public void finishBundle(DoFn<Read, String>.FinishBundleContext c) throws IOException {
  bw.close();
  Metrics.distribution(WriteBAMFn.class, "Maximum Write Shard Processing Time (sec)")
      .update(stopWatch.elapsed(TimeUnit.SECONDS));
  LOG.info("Finished writing " + shardContig);
  Metrics.counter(WriteBAMFn.class, "Finished Write Shard Count").inc();
  final long bytesWritten = ts.getBytesWrittenExceptingTruncation();
  LOG.info("Wrote " + readCount + " reads, " + unmappedReadCount + " unmapped, into " + shardName +
      (hadOutOfOrder ? "ignored out of order" : "") + ", wrote " + bytesWritten + " bytes");
  Metrics.counter(WriteBAMFn.class, "Written reads").inc(readCount);
  Metrics.counter(WriteBAMFn.class, "Written unmapped reads").inc(unmappedReadCount);
  final long totalReadCount = (long)readCount + (long)unmappedReadCount;
  Metrics.distribution(WriteBAMFn.class, "Maximum Reads Per Shard").update(totalReadCount);
  c.output(shardName, window.maxTimestamp(), window);
  c.output(SEQUENCE_SHARD_SIZES_TAG, KV.of(sequenceIndex, bytesWritten),
      window.maxTimestamp(), window);
}
 

@ProcessElement
public void processElement(DoFn<String, Integer>.ProcessContext context) throws Exception {
  String variantId = context.element();
  // Call the deletion operation via exponential backoff so that "Rate Limit Exceeded"
  // quota issues do not cause the pipeline to fail.
  ExponentialBackOff backoff = new ExponentialBackOff.Builder().build();
  while (true) {
    try {
      genomics.variants().delete(variantId).execute();
      Metrics.counter(DeleteVariantFn.class, "Number of variants deleted").inc();
      context.output(1);
      return;
    } catch (Exception e) {
      if (e.getMessage().startsWith("429 Too Many Requests")) {
        LOG.warn("Backing-off per: ", e);
        long backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
          throw e;
        }
        Thread.sleep(backOffMillis);
      } else {
        throw e;
      }
    }
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@ProcessElement
public void processElement(DoFn<Read, KV<Contig, Read>>.ProcessContext c)
  throws Exception {
  final Read read = c.element();
  long pos = read.getAlignment().getPosition().getPosition();
  minPos = Math.min(minPos, pos);
  maxPos = Math.max(maxPos, pos);
  count++;
  c.output(
      KV.of(
          shardKeyForRead(read, lociPerShard),
          read));
  Metrics.counter(KeyReadsFn.class, "Keyed reads").inc();
  if (isUnmapped(read)) {
    Metrics.counter(KeyReadsFn.class, "Keyed unmapped reads").inc();
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
    throw new RuntimeException( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", counterName );
    writtenCounter = Metrics.counter( "written", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init(stepPluginClasses, xpPluginClasses);
    groupRowMeta = JsonRowMeta.fromJson( groupRowMetaJson );
    subjectRowMeta = JsonRowMeta.fromJson( subjectRowMetaJson );
    aggregationTypes = new AggregationType[aggregations.length];
    for ( int i = 0; i < aggregationTypes.length; i++ ) {
      aggregationTypes[ i ] = AggregationType.getTypeFromName( aggregations[ i ] );
    }

    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of grouping by ", e);
    throw new RuntimeException( "Unable setup of group by ", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    outputCounter = Metrics.counter( "output", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    outputCounter = Metrics.counter( "output", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", counterName );
    outputCounter = Metrics.counter( "output", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", counterName ).inc();
  } catch ( Exception e ) {
    errorCounter.inc();
    LOG.info( "Parse error on setup of Kettle data to string lines : " + e.getMessage() );
    throw new RuntimeException( "Error on setup of converting Kettle data to string lines", e );
  }
}
 

public NexmarkQuery(NexmarkConfiguration configuration, NexmarkQueryTransform<T> transform) {
  super(transform.getName());
  this.configuration = configuration;
  this.transform = transform;
  if (configuration.debug) {
    eventMonitor = new Monitor<>(name + ".Events", "event");
    resultMonitor = new Monitor<>(name + ".Results", "result");
    endOfStreamMonitor = new Monitor<>(name + ".EndOfStream", "end");
    fatalCounter = Metrics.counter(name, "fatal");
  } else {
    eventMonitor = null;
    resultMonitor = null;
    endOfStreamMonitor = null;
    fatalCounter = null;
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
    throw new RuntimeException( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
  }
}
 

private PTransform<? super PCollection<Row>, PCollection<Row>> logBytesMetric(
    final Coder<Row> coder) {

  return ParDo.of(
      new DoFn<Row, Row>() {
        private final Counter bytesMetric = Metrics.counter(name, "bytes");

        @ProcessElement
        public void processElement(@Element Row element, OutputReceiver<Row> o)
            throws IOException {
          ByteArrayOutputStream outStream = new ByteArrayOutputStream();
          coder.encode(element, outStream, Coder.Context.OUTER);
          byte[] byteArray = outStream.toByteArray();
          bytesMetric.inc((long) byteArray.length);
          ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
          Row row = coder.decode(inStream, Coder.Context.OUTER);
          o.output(row);
        }
      });
}
 

@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    writtenCounter = Metrics.counter( "written", stepname );
    errorCounter = Metrics.counter( "error", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    inputRowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error in setup of adding window information to rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of adding window information to rows", e );
  }
}
 

@Setup
public void setUp() {
  try {
    writtenCounter = Metrics.counter( "written", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    outputRowMeta = JsonRowMeta.fromJson( outputRowMetaJson );
    leftKRowMeta = JsonRowMeta.fromJson( leftKRowMetaJson );
    leftVRowMeta = JsonRowMeta.fromJson( leftVRowMetaJson );
    rightVRowMeta = JsonRowMeta.fromJson( rightVRowMetaJson );

    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error initializing assembling rows", e);
    throw new RuntimeException( "Error initializing assembling output KV<row, KV<row, row>>", e );
  }
}
 

@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    Metrics.counter( "error", stepname ).inc();
    LOG.error( "Error in setup of converting input data into Kettle rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Kettle rows", e );
  }
}
 

@Override
public PCollection<Event> expand(PCollection<Event> events) {
  final Coder<Event> coder = events.getCoder();
  return events
      // Force round trip through coder.
      .apply(
      name + ".Serialize",
      ParDo.of(
          new DoFn<Event, Event>() {
            private final Counter bytesMetric = Metrics.counter(name, "bytes");

            @ProcessElement
            public void processElement(ProcessContext c) throws CoderException, IOException {
              ByteArrayOutputStream outStream = new ByteArrayOutputStream();
              coder.encode(c.element(), outStream, Coder.Context.OUTER);
              byte[] byteArray = outStream.toByteArray();
              bytesMetric.inc((long) byteArray.length);
              ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
              Event event = coder.decode(inStream, Coder.Context.OUTER);
              c.output(event);
            }
          }));
}