Java Code Examples for org.apache.beam.sdk.runners.AppliedPTransform#getFullName()

The following examples show how to use org.apache.beam.sdk.runners.AppliedPTransform#getFullName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JetTransformTranslators.java From beam with Apache License 2.0

6 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    Node node,
    JetTranslationContext context) {
  String transformName = appliedTransform.getFullName();
  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);

  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder =
      Utils.getCoder((PCollection) Utils.getOutput(appliedTransform).getValue());
  Vertex vertex = dagBuilder.addVertex(vertexId, ImpulseP.supplier(outputCoder, vertexId));

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}

Example 2

Source File: SdkComponents.java From beam with Apache License 2.0

5 votes

/**
 * Gets the ID for the provided {@link AppliedPTransform}. The provided {@link AppliedPTransform}
 * will not be added to the components produced by this {@link SdkComponents} until it is
 * translated via {@link #registerPTransform(AppliedPTransform, List)}.
 */
private String getApplicationName(AppliedPTransform<?, ?, ?> appliedPTransform) {
  String existing = transformIds.get(appliedPTransform);
  if (existing != null) {
    return existing;
  }

  String name = appliedPTransform.getFullName();
  if (name.isEmpty()) {
    name = "unnamed-ptransform";
  }
  name = uniqify(name, transformIds.values());
  transformIds.put(appliedPTransform, name);
  return name;
}

Example 3

Source File: JetTransformTranslators.java From beam with Apache License 2.0

5 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    Node node,
    JetTranslationContext context) {
  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder =
      Utils.getCoder((PCollection) Utils.getOutput(appliedTransform).getValue());

  String transformName = appliedTransform.getFullName();
  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);
  SerializablePipelineOptions pipelineOptions = context.getOptions();
  ProcessorMetaSupplier processorSupplier =
      getProcessorSupplier(
          (AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>>)
              appliedTransform,
          outputCoder,
          vertexId,
          pipelineOptions);

  Vertex vertex = dagBuilder.addVertex(vertexId, processorSupplier);

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}

Example 4

Source File: JetTransformTranslators.java From beam with Apache License 2.0

5 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    Node node,
    JetTranslationContext context) {
  String transformName = appliedTransform.getFullName();

  PCollection<KV<K, InputT>> input =
      (PCollection<KV<K, InputT>>) Utils.getInput(appliedTransform);
  WindowedValue.WindowedValueCoder<KV<K, InputT>> inputCoder =
      Utils.getWindowedValueCoder(input);
  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder = Utils.getCoder((PCollection) output.getValue());

  WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();

  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);
  Vertex vertex =
      dagBuilder.addVertex(
          vertexId,
          WindowGroupP.supplier(
              context.getOptions(), inputCoder, outputCoder, windowingStrategy, vertexId));

  dagBuilder.registerEdgeEndPoint(Utils.getTupleTagId(input), vertex);

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}

Example 5

Source File: JetTransformTranslators.java From beam with Apache License 2.0

5 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    Node node,
    JetTranslationContext context) {
  WindowingStrategy<T, BoundedWindow> windowingStrategy =
      (WindowingStrategy<T, BoundedWindow>)
          ((PCollection) Utils.getOutput(appliedTransform).getValue()).getWindowingStrategy();

  PCollection<WindowedValue> input =
      (PCollection<WindowedValue>) Utils.getInput(appliedTransform);
  Coder inputCoder = Utils.getCoder(input);
  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder =
      Utils.getCoder((PCollection) Utils.getOutput(appliedTransform).getValue());

  String transformName = appliedTransform.getFullName();
  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);

  Vertex vertex =
      dagBuilder.addVertex(
          vertexId,
          AssignWindowP.supplier(inputCoder, outputCoder, windowingStrategy, vertexId));
  dagBuilder.registerEdgeEndPoint(Utils.getTupleTagId(input), vertex);

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}

Example 6

Source File: TestJetRunner.java From beam with Apache License 2.0

5 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    TransformHierarchy.Node node,
    JetTranslationContext context) {
  String transformName = appliedTransform.getFullName();
  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);

  TestStream<T> testStream = (TestStream<T>) appliedTransform.getTransform();

  // events in the transform are not serializable, we have to translate them. We'll also flatten
  // the collection.
  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder = Utils.getCoder((PCollection) output.getValue());
  TestStream.TestStreamCoder<T> payloadCoder =
      TestStream.TestStreamCoder.of(testStream.getValueCoder());
  byte[] encodedPayload = getEncodedPayload(testStream, payloadCoder);
  Vertex vertex =
      dagBuilder.addVertex(
          vertexId, TestStreamP.supplier(encodedPayload, payloadCoder, outputCoder));

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}

Example 7

Source File: JetTransformTranslators.java From beam with Apache License 2.0

4 votes

@Override
public Vertex translate(
    Pipeline pipeline,
    AppliedPTransform<?, ?, ?> appliedTransform,
    Node node,
    JetTranslationContext context) {
  PCollectionView<T> view;
  try {
    view =
        CreatePCollectionViewTranslation.getView(
            (AppliedPTransform<
                    PCollection<T>, PCollection<T>, PTransform<PCollection<T>, PCollection<T>>>)
                appliedTransform);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }

  String transformName = appliedTransform.getFullName();
  DAGBuilder dagBuilder = context.getDagBuilder();
  String vertexId = dagBuilder.newVertexId(transformName);
  PCollection<T> input = (PCollection<T>) Utils.getInput(appliedTransform);
  Coder inputCoder = Utils.getCoder(input);
  Map.Entry<TupleTag<?>, PValue> output = Utils.getOutput(appliedTransform);
  Coder outputCoder = Utils.getCoder((PCollection) output.getValue());

  Vertex vertex =
      dagBuilder.addVertex(
          vertexId,
          ViewP.supplier(inputCoder, outputCoder, input.getWindowingStrategy(), vertexId));

  dagBuilder.registerEdgeEndPoint(Utils.getTupleTagId(input), vertex);

  String viewTag = Utils.getTupleTagId(view);
  dagBuilder.registerSideInput(view);
  dagBuilder.registerCollectionOfEdge(viewTag, view.getTagInternal().getId());
  dagBuilder.registerEdgeStartPoint(viewTag, vertex, outputCoder);

  String outputEdgeId = Utils.getTupleTagId(output.getValue());
  dagBuilder.registerCollectionOfEdge(outputEdgeId, output.getKey().getId());
  dagBuilder.registerEdgeStartPoint(outputEdgeId, vertex, outputCoder);
  return vertex;
}