org.apache.beam.sdk.metrics.Metrics Java Examples

The following examples show how to use org.apache.beam.sdk.metrics.Metrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KettleRowToKVStringStringFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KettleRow to KV<String,String> function", e );
    throw new RuntimeException( "Error in setup of KettleRow to KV<String,String> function", e );
  }
}
 
Example #2
Source File: HopToStringFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, counterName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch ( Exception e ) {
    errorCounter.inc();
    LOG.info( "Parse error on setup of Hop data to string lines : " + e.getMessage() );
    throw new RuntimeException( "Error on setup of converting Hop data to string lines", e );
  }
}
 
Example #3
Source File: HopRowToKVStringStringFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of HopRow to KV<String,String> function", e );
    throw new RuntimeException( "Error in setup of HopRow to KV<String,String> function", e );
  }
}
 
Example #4
Source File: PublishStringsFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #5
Source File: PublishMessagesFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, transformName );
    outputCounter = Metrics.counter( Pipeline.METRIC_NAME_OUTPUT, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #6
Source File: GroupByFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( Pipeline.METRIC_NAME_READ, counterName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, counterName );
    errorCounter = Metrics.counter( Pipeline.METRIC_NAME_ERROR, counterName );

    // Initialize Hop Beam
    //
    BeamHop.init(transformPluginClasses, xpPluginClasses);
    groupRowMeta = JsonRowMeta.fromJson( groupRowMetaJson );
    subjectRowMeta = JsonRowMeta.fromJson( subjectRowMetaJson );
    aggregationTypes = new AggregationType[aggregations.length];
    for ( int i = 0; i < aggregationTypes.length; i++ ) {
      aggregationTypes[ i ] = AggregationType.getTypeFromName( aggregations[ i ] );
    }

    Metrics.counter( Pipeline.METRIC_NAME_INIT, counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of grouping by ", e);
    throw new RuntimeException( "Unable setup of group by ", e );
  }
}
 
Example #7
Source File: GenomicsApiReader.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {
  GenomicsFactory factory = GenomicsFactory.builder().build();
  Genomics genomics = factory.fromOfflineAuth(auth);

  processApiCall(genomics, c, c.element());

  Metrics.counter(GenomicsApiReader.class, "Genomics API Initialized Request Count")
      .inc(factory.initializedRequestsCount());
  Metrics.counter(GenomicsApiReader.class, "Genomics API Unsuccessful Response Count")
      .inc(factory.unsuccessfulResponsesCount());
  Metrics.counter(GenomicsApiReader.class, "Genomics API IOException Response Count")
      .inc(factory.ioExceptionsCount());
  LOG.info("ApiReader processed " + factory.initializedRequestsCount() + " requests ("
      + factory.unsuccessfulResponsesCount() + " server errors and "
      + factory.ioExceptionsCount() + " IO exceptions)");
}
 
Example #8
Source File: StringToHopFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    Metrics.counter( Pipeline.METRIC_NAME_ERROR, transformName ).inc();
    LOG.error( "Error in setup of converting input data into Hop rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Hop rows", e );
  }
}
 
Example #9
Source File: BeamJdbcAvroSchema.java    From dbeam with Apache License 2.0 6 votes vote down vote up
/** Generate Avro schema by reading one row. Expose Beam metrics via a Beam PTransform. */
public static Schema createSchema(
    final Pipeline pipeline, final JdbcExportArgs args, final Connection connection)
    throws Exception {
  final long startTime = System.nanoTime();
  final Schema generatedSchema = generateAvroSchema(args, connection);
  final long elapsedTimeSchema = (System.nanoTime() - startTime) / 1000000;
  LOGGER.info("Elapsed time to schema {} seconds", elapsedTimeSchema / 1000.0);

  final Counter cnt =
      Metrics.counter(BeamJdbcAvroSchema.class.getCanonicalName(), "schemaElapsedTimeMs");
  pipeline
      .apply(
          "ExposeSchemaCountersSeed",
          Create.of(Collections.singletonList(0)).withType(TypeDescriptors.integers()))
      .apply(
          "ExposeSchemaCounters",
          MapElements.into(TypeDescriptors.integers())
              .via(
                  v -> {
                    cnt.inc(elapsedTimeSchema);
                    return v;
                  }));
  return generatedSchema;
}
 
Example #10
Source File: VariantStreamer.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) throws IOException, GeneralSecurityException, InterruptedException {
  Metrics.counter(RetrieveVariants.class, "Initialized Shard Count").inc();
  Stopwatch stopWatch = Stopwatch.createStarted();
  Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields);
  while (iter.hasNext()) {
    StreamVariantsResponse variantResponse = iter.next();
    c.output(variantResponse.getVariantsList());
  }
  stopWatch.stop();
  Metrics.distribution(RetrieveVariants.class, "Shard Processing Time (sec)")
      .update(stopWatch.elapsed(TimeUnit.SECONDS));
  Metrics.counter(RetrieveVariants.class, "Finished Shard Count").inc();
  stats.addValue(stopWatch.elapsed(TimeUnit.SECONDS));
  LOG.info("Shard Duration in Seconds - Min: " + stats.getMin() + " Max: " + stats.getMax() +
      " Avg: " + stats.getMean() + " StdDev: " + stats.getStandardDeviation());
}
 
Example #11
Source File: StringToHopRowFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch(Exception e) {
    numErrors.inc();
    LOG.error( "Error in setup of String to Hop Row conversion function", e );
    throw new RuntimeException( "Error in setup of String to Hop Row conversion function", e );
  }
}
 
Example #12
Source File: PerDocTypeCounter.java    From gcp-ingestion with Mozilla Public License 2.0 6 votes vote down vote up
@VisibleForTesting
static Counter getOrCreateCounter(@Nullable Map<String, String> attributes, String name) {
  if (attributes == null) {
    attributes = EMPTY_ATTRIBUTES;
  }
  String namespace = attributes.getOrDefault("document_namespace", "unknown_namespace");
  // Dataflow's UI collapses the metric name, but always shows the penultimate path component
  // as part of "Step", so we're a bit more verbose with the default doctype value.
  String docType = attributes.getOrDefault("document_type", "unknown_doctype");
  String version = attributes.get("document_version");
  if (version != null) {
    docType = docType + "_v" + version;
  }
  String key = String.format("%s/%s/%s", namespace, docType, name)
      // We change dashes to underscores to make sure we're following Stackdriver's naming scheme:
      // https://cloud.google.com/monitoring/api/v3/metrics-details#label_names
      .replace("-", "_");
  return counters.computeIfAbsent(key, k -> Metrics.counter(PerDocTypeCounter.class, k));
}
 
Example #13
Source File: WriteBAMFn.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@StartBundle
public void startBundle(DoFn<Read, String>.StartBundleContext c) throws IOException {
  LOG.info("Starting bundle ");
  storage = Transport.newStorageClient(c.getPipelineOptions().as(GCSOptions.class)).build().objects();

  Metrics.counter(WriteBAMFn.class, "Initialized Write Shard Count").inc();
  stopWatch = Stopwatch.createStarted();

  options = c.getPipelineOptions().as(Options.class);

  readCount = 0;
  unmappedReadCount = 0;
  headerInfo = null;
  prevRead = null;
  minAlignment = Long.MAX_VALUE;
  maxAlignment = Long.MIN_VALUE;
  hadOutOfOrder = false;
}
 
Example #14
Source File: WriteBAMFn.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@FinishBundle
public void finishBundle(DoFn<Read, String>.FinishBundleContext c) throws IOException {
  bw.close();
  Metrics.distribution(WriteBAMFn.class, "Maximum Write Shard Processing Time (sec)")
      .update(stopWatch.elapsed(TimeUnit.SECONDS));
  LOG.info("Finished writing " + shardContig);
  Metrics.counter(WriteBAMFn.class, "Finished Write Shard Count").inc();
  final long bytesWritten = ts.getBytesWrittenExceptingTruncation();
  LOG.info("Wrote " + readCount + " reads, " + unmappedReadCount + " unmapped, into " + shardName +
      (hadOutOfOrder ? "ignored out of order" : "") + ", wrote " + bytesWritten + " bytes");
  Metrics.counter(WriteBAMFn.class, "Written reads").inc(readCount);
  Metrics.counter(WriteBAMFn.class, "Written unmapped reads").inc(unmappedReadCount);
  final long totalReadCount = (long)readCount + (long)unmappedReadCount;
  Metrics.distribution(WriteBAMFn.class, "Maximum Reads Per Shard").update(totalReadCount);
  c.output(shardName, window.maxTimestamp(), window);
  c.output(SEQUENCE_SHARD_SIZES_TAG, KV.of(sequenceIndex, bytesWritten),
      window.maxTimestamp(), window);
}
 
Example #15
Source File: DeleteVariants.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(DoFn<String, Integer>.ProcessContext context) throws Exception {
  String variantId = context.element();
  // Call the deletion operation via exponential backoff so that "Rate Limit Exceeded"
  // quota issues do not cause the pipeline to fail.
  ExponentialBackOff backoff = new ExponentialBackOff.Builder().build();
  while (true) {
    try {
      genomics.variants().delete(variantId).execute();
      Metrics.counter(DeleteVariantFn.class, "Number of variants deleted").inc();
      context.output(1);
      return;
    } catch (Exception e) {
      if (e.getMessage().startsWith("429 Too Many Requests")) {
        LOG.warn("Backing-off per: ", e);
        long backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
          throw e;
        }
        Thread.sleep(backOffMillis);
      } else {
        throw e;
      }
    }
  }
}
 
Example #16
Source File: PubsubMessageToHopRowFn.java    From hop with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( Pipeline.METRIC_NAME_INPUT, transformName );
    writtenCounter = Metrics.counter( Pipeline.METRIC_NAME_WRITTEN, transformName );

    // Initialize Hop Beam
    //
    BeamHop.init( transformPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( Pipeline.METRIC_NAME_INIT, transformName ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #17
Source File: KeyReadsFn.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(DoFn<Read, KV<Contig, Read>>.ProcessContext c)
  throws Exception {
  final Read read = c.element();
  long pos = read.getAlignment().getPosition().getPosition();
  minPos = Math.min(minPos, pos);
  maxPos = Math.max(maxPos, pos);
  count++;
  c.output(
      KV.of(
          shardKeyForRead(read, lociPerShard),
          read));
  Metrics.counter(KeyReadsFn.class, "Keyed reads").inc();
  if (isUnmapped(read)) {
    Metrics.counter(KeyReadsFn.class, "Keyed unmapped reads").inc();
  }
}
 
Example #18
Source File: KVLongStringToKettleRowFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
    throw new RuntimeException( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
  }
}
 
Example #19
Source File: GroupByFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", counterName );
    writtenCounter = Metrics.counter( "written", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init(stepPluginClasses, xpPluginClasses);
    groupRowMeta = JsonRowMeta.fromJson( groupRowMetaJson );
    subjectRowMeta = JsonRowMeta.fromJson( subjectRowMetaJson );
    aggregationTypes = new AggregationType[aggregations.length];
    for ( int i = 0; i < aggregationTypes.length; i++ ) {
      aggregationTypes[ i ] = AggregationType.getTypeFromName( aggregations[ i ] );
    }

    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error("Error setup of grouping by ", e);
    throw new RuntimeException( "Unable setup of group by ", e );
  }
}
 
Example #20
Source File: PublishMessagesFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    outputCounter = Metrics.counter( "output", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #21
Source File: PubsubMessageToKettleRowFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #22
Source File: PublishStringsFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    outputCounter = Metrics.counter( "output", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of pub/sub publish messages function", e );
    throw new RuntimeException( "Error in setup of pub/sub publish messages function", e );
  }
}
 
Example #23
Source File: KettleToStringFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", counterName );
    outputCounter = Metrics.counter( "output", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", counterName ).inc();
  } catch ( Exception e ) {
    errorCounter.inc();
    LOG.info( "Parse error on setup of Kettle data to string lines : " + e.getMessage() );
    throw new RuntimeException( "Error on setup of converting Kettle data to string lines", e );
  }
}
 
Example #24
Source File: NexmarkQuery.java    From beam with Apache License 2.0 6 votes vote down vote up
public NexmarkQuery(NexmarkConfiguration configuration, NexmarkQueryTransform<T> transform) {
  super(transform.getName());
  this.configuration = configuration;
  this.transform = transform;
  if (configuration.debug) {
    eventMonitor = new Monitor<>(name + ".Events", "event");
    resultMonitor = new Monitor<>(name + ".Results", "result");
    endOfStreamMonitor = new Monitor<>(name + ".EndOfStream", "end");
    fatalCounter = Metrics.counter(name, "fatal");
  } else {
    eventMonitor = null;
    resultMonitor = null;
    endOfStreamMonitor = null;
    fatalCounter = null;
  }
}
 
Example #25
Source File: KVStringStringToKettleRowFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    numErrors.inc();
    LOG.error( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
    throw new RuntimeException( "Error in setup of KV<Long,String> to Kettle Row conversion function", e );
  }
}
 
Example #26
Source File: SqlQuery0.java    From beam with Apache License 2.0 6 votes vote down vote up
private PTransform<? super PCollection<Row>, PCollection<Row>> logBytesMetric(
    final Coder<Row> coder) {

  return ParDo.of(
      new DoFn<Row, Row>() {
        private final Counter bytesMetric = Metrics.counter(name, "bytes");

        @ProcessElement
        public void processElement(@Element Row element, OutputReceiver<Row> o)
            throws IOException {
          ByteArrayOutputStream outStream = new ByteArrayOutputStream();
          coder.encode(element, outStream, Coder.Context.OUTER);
          byte[] byteArray = outStream.toByteArray();
          bytesMetric.inc((long) byteArray.length);
          ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
          Row row = coder.decode(inStream, Coder.Context.OUTER);
          o.output(row);
        }
      });
}
 
Example #27
Source File: WindowInfoFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    readCounter = Metrics.counter( "read", stepname );
    writtenCounter = Metrics.counter( "written", stepname );
    errorCounter = Metrics.counter( "error", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    inputRowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error in setup of adding window information to rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of adding window information to rows", e );
  }
}
 
Example #28
Source File: AssemblerFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    writtenCounter = Metrics.counter( "written", counterName );
    errorCounter = Metrics.counter( "error", counterName );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    outputRowMeta = JsonRowMeta.fromJson( outputRowMetaJson );
    leftKRowMeta = JsonRowMeta.fromJson( leftKRowMetaJson );
    leftVRowMeta = JsonRowMeta.fromJson( leftVRowMetaJson );
    rightVRowMeta = JsonRowMeta.fromJson( rightVRowMetaJson );

    Metrics.counter( "init", counterName ).inc();
  } catch(Exception e) {
    errorCounter.inc();
    LOG.error( "Error initializing assembling rows", e);
    throw new RuntimeException( "Error initializing assembling output KV<row, KV<row, row>>", e );
  }
}
 
Example #29
Source File: StringToKettleFn.java    From kettle-beam with Apache License 2.0 6 votes vote down vote up
@Setup
public void setUp() {
  try {
    inputCounter = Metrics.counter( "input", stepname );
    writtenCounter = Metrics.counter( "written", stepname );

    // Initialize Kettle Beam
    //
    BeamKettle.init( stepPluginClasses, xpPluginClasses );
    rowMeta = JsonRowMeta.fromJson( rowMetaJson );

    Metrics.counter( "init", stepname ).inc();
  } catch ( Exception e ) {
    Metrics.counter( "error", stepname ).inc();
    LOG.error( "Error in setup of converting input data into Kettle rows : " + e.getMessage() );
    throw new RuntimeException( "Error in setup of converting input data into Kettle rows", e );
  }
}
 
Example #30
Source File: Query0.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PCollection<Event> expand(PCollection<Event> events) {
  final Coder<Event> coder = events.getCoder();
  return events
      // Force round trip through coder.
      .apply(
      name + ".Serialize",
      ParDo.of(
          new DoFn<Event, Event>() {
            private final Counter bytesMetric = Metrics.counter(name, "bytes");

            @ProcessElement
            public void processElement(ProcessContext c) throws CoderException, IOException {
              ByteArrayOutputStream outStream = new ByteArrayOutputStream();
              coder.encode(c.element(), outStream, Coder.Context.OUTER);
              byte[] byteArray = outStream.toByteArray();
              bytesMetric.inc((long) byteArray.length);
              ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
              Event event = coder.decode(inStream, Coder.Context.OUTER);
              c.output(event);
            }
          }));
}