org.apache.beam.sdk.runners.AppliedPTransform Java Examples

The following examples show how to use org.apache.beam.sdk.runners.AppliedPTransform. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PTransformMatchers.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * A {@link PTransformMatcher} that matches a {@link ParDo.SingleOutput} containing a {@link DoFn}
 * that is splittable, as signified by {@link ProcessElementMethod#isSplittable()}.
 */
public static PTransformMatcher splittableParDoSingle() {
  return new PTransformMatcher() {
    @Override
    public boolean matches(AppliedPTransform<?, ?, ?> application) {
      PTransform<?, ?> transform = application.getTransform();
      if (transform instanceof ParDo.SingleOutput) {
        DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn();
        DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn);
        return signature.processElement().isSplittable();
      }
      return false;
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper("SplittableParDoSingleMatcher").toString();
    }
  };
}
 
Example #2
Source File: CreatePCollectionViewTranslationTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testExtractionDirectFromTransform() throws Exception {
  SdkComponents components = SdkComponents.create();
  components.registerEnvironment(Environments.createDockerEnvironment("java"));
  components.registerPCollection(testPCollection);

  AppliedPTransform<?, ?, ?> appliedPTransform =
      AppliedPTransform.of(
          "foo",
          testPCollection.expand(),
          createViewTransform.getView().expand(),
          createViewTransform,
          p);

  CreatePCollectionViewTranslation.getView((AppliedPTransform) appliedPTransform);

  FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec();

  // Checks that the payload is what it should be
  PCollectionView<?> deserializedView =
      (PCollectionView<?>)
          SerializableUtils.deserializeFromByteArray(
              payload.getPayload().toByteArray(), PCollectionView.class.getSimpleName());

  assertThat(deserializedView, Matchers.equalTo(createViewTransform.getView()));
}
 
Example #3
Source File: DirectTransformExecutor.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
DirectTransformExecutor(
    EvaluationContext context,
    TransformEvaluatorRegistry factory,
    Iterable<? extends ModelEnforcementFactory> modelEnforcements,
    CommittedBundle<T> inputBundle,
    AppliedPTransform<?, ?, ?> transform,
    CompletionCallback completionCallback,
    TransformExecutorService transformEvaluationState) {
  this.evaluatorRegistry = factory;
  this.modelEnforcements = modelEnforcements;

  this.inputBundle = inputBundle;
  this.transform = transform;

  this.onComplete = completionCallback;

  this.transformEvaluationState = transformEvaluationState;
  this.context = context;
}
 
Example #4
Source File: WriteFilesTranslationTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testExtractionDirectFromTransform() throws Exception {
  PCollection<String> input = p.apply(Create.of("hello"));
  WriteFilesResult<Void> output = input.apply(writeFiles);

  AppliedPTransform<PCollection<String>, WriteFilesResult<Void>, WriteFiles<String, Void, String>>
      appliedPTransform =
          AppliedPTransform.of("foo", input.expand(), output.expand(), writeFiles, p);

  assertThat(
      WriteFilesTranslation.isRunnerDeterminedSharding(appliedPTransform),
      equalTo(
          writeFiles.getNumShardsProvider() == null && writeFiles.getComputeNumShards() == null));

  assertThat(
      WriteFilesTranslation.isWindowedWrites(appliedPTransform),
      equalTo(writeFiles.getWindowedWrites()));
  assertThat(
      WriteFilesTranslation.<String, Void, String>getSink(appliedPTransform),
      equalTo(writeFiles.getSink()));
}
 
Example #5
Source File: PTransformTranslationTest.java    From beam with Apache License 2.0 6 votes vote down vote up
private static AppliedPTransform<?, ?, ?> multiMultiParDo(Pipeline pipeline) {
  PCollectionView<String> view = pipeline.apply(Create.of("foo")).apply(View.asSingleton());
  PCollection<Long> input = pipeline.apply(GenerateSequence.from(0));
  ParDo.MultiOutput<Long, KV<Long, String>> parDo =
      ParDo.of(new TestDoFn())
          .withSideInputs(view)
          .withOutputTags(
              new TupleTag<KV<Long, String>>() {},
              TupleTagList.of(new TupleTag<KV<String, Long>>() {}));
  PCollectionTuple output = input.apply(parDo);

  Map<TupleTag<?>, PValue> inputs = new HashMap<>();
  inputs.putAll(parDo.getAdditionalInputs());
  inputs.putAll(input.expand());

  return AppliedPTransform
      .<PCollection<Long>, PCollectionTuple, ParDo.MultiOutput<Long, KV<Long, String>>>of(
          "MultiParDoInAndOut", inputs, output.expand(), parDo, pipeline);
}
 
Example #6
Source File: ParDoTranslation.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public RunnerApi.PTransform translate(
    AppliedPTransform<?, ?, ?> appliedPTransform,
    List<AppliedPTransform<?, ?, ?>> subtransforms,
    SdkComponents components)
    throws IOException {
  RunnerApi.PTransform.Builder builder =
      PTransformTranslation.translateAppliedPTransform(
          appliedPTransform, subtransforms, components);

  AppliedPTransform<?, ?, ParDo.MultiOutput<?, ?>> appliedParDo =
      (AppliedPTransform<?, ?, ParDo.MultiOutput<?, ?>>) appliedPTransform;
  ParDoPayload payload = translateParDo(appliedParDo, components);
  builder.setSpec(
      RunnerApi.FunctionSpec.newBuilder()
          .setUrn(PAR_DO_TRANSFORM_URN)
          .setPayload(payload.toByteString())
          .build());
  builder.setEnvironmentId(components.getOnlyEnvironmentId());

  return builder.build();
}
 
Example #7
Source File: ViewOverrideFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PTransformReplacement<PCollection<ElemT>, PCollection<ElemT>> getReplacementTransform(
    AppliedPTransform<
            PCollection<ElemT>,
            PCollection<ElemT>,
            PTransform<PCollection<ElemT>, PCollection<ElemT>>>
        transform) {

  PCollectionView<ViewT> view;
  try {
    view = CreatePCollectionViewTranslation.getView(transform);
  } catch (IOException exc) {
    throw new RuntimeException(
        String.format(
            "Could not extract %s from transform %s",
            PCollectionView.class.getSimpleName(), transform),
        exc);
  }

  return PTransformReplacement.of(
      PTransformReplacements.getSingletonMainInput(transform), new GroupAndWriteView<>(view));
}
 
Example #8
Source File: DataflowPTransformMatchersTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Test significant cases that the matcher should not match against. In this case, this tests that
 * any {@link Combine.GroupedValues} with side inputs should not match, and that a {@link
 * Combine.GroupedValues} without an encompassing {@link Combine.PerKey} will not match.
 */
@Test
public void combineValuesWithParentCheckSkipsNonmatching() {
  PTransformMatcher matcher =
      new DataflowPTransformMatchers.CombineValuesWithParentCheckPTransformMatcher();
  AppliedPTransform<?, ?, ?> groupedValues;

  groupedValues = getCombineGroupedValuesFrom(createCombineGroupedValuesPipeline());
  assertThat(matcher.matches(groupedValues), is(false));

  groupedValues = getCombineGroupedValuesFrom(createCombineGroupedValuesWithSideInputsPipeline());
  assertThat(matcher.matches(groupedValues), is(false));

  groupedValues = getCombineGroupedValuesFrom(createCombinePerKeyWithSideInputsPipeline());
  assertThat(matcher.matches(groupedValues), is(false));
}
 
Example #9
Source File: CombineTranslation.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<KV<K, InputT>> mainInput =
        (PCollection<KV<K, InputT>>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(),
        ((KvCoder<K, InputT>) mainInput.getCoder()).getValueCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 
Example #10
Source File: TestStreamEvaluatorFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public Collection<CommittedBundle<TestStreamIndex<T>>> getInitialInputs(
    AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform,
    int targetParallelism) {

  // This will always be run on an execution-time transform, so it can be downcast
  DirectTestStreamFactory.DirectTestStream<T> testStream =
      (DirectTestStreamFactory.DirectTestStream<T>) transform.getTransform();

  CommittedBundle<TestStreamIndex<T>> initialBundle =
      evaluationContext
          .<TestStreamIndex<T>>createRootBundle()
          .add(WindowedValue.valueInGlobalWindow(TestStreamIndex.of(testStream.original)))
          .commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
  return Collections.singleton(initialBundle);
}
 
Example #11
Source File: DirectGraphVisitorTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void getValueToConsumersWithDuplicateInputSucceeds() {
  PCollection<String> created = p.apply(Create.of("1", "2", "3"));

  PCollection<String> flattened =
      PCollectionList.of(created).and(created).apply(Flatten.pCollections());

  p.traverseTopologically(visitor);

  DirectGraph graph = visitor.getGraph();
  AppliedPTransform<?, ?, ?> flattenedProducer = graph.getProducer(flattened);

  assertThat(
      graph.getPerElementConsumers(created),
      Matchers.containsInAnyOrder(new Object[] {flattenedProducer, flattenedProducer}));
  assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
 
Example #12
Source File: Utils.java    From beam with Apache License 2.0 6 votes vote down vote up
static DoFn<?, ?> getDoFn(AppliedPTransform<?, ?, ?> appliedTransform) {
  try {
    DoFn<?, ?> doFn = ParDoTranslation.getDoFn(appliedTransform);
    if (DoFnSignatures.isSplittable(doFn)) {
      throw new IllegalStateException(
          "Not expected to directly translate splittable DoFn, should have been overridden: "
              + doFn); // todo
    }
    if (DoFnSignatures.requiresTimeSortedInput(doFn)) {
      throw new UnsupportedOperationException(
          String.format(
              "%s doesn't current support @RequiresTimeSortedInput annotation.",
              JetRunner.class.getSimpleName()));
    }
    return doFn;
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #13
Source File: SdkComponentsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that trying to register a transform which has unregistered children throws. */
@Test
public void registerTransformWithUnregisteredChildren() throws IOException {
  Create.Values<Long> create = Create.of(1L, 2L, 3L);
  GenerateSequence createChild = GenerateSequence.from(0);

  PCollection<Long> pt = pipeline.apply(create);
  String userName = "my_transform";
  String childUserName = "my_transform/my_nesting";
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
  AppliedPTransform<?, ?, ?> childTransform =
      AppliedPTransform.of(
          childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(childTransform.toString());
  components.registerPTransform(transform, Collections.singletonList(childTransform));
}
 
Example #14
Source File: DataflowRunnerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
private void testStreamingWriteOverride(PipelineOptions options, int expectedNumShards) {
  TestPipeline p = TestPipeline.fromOptions(options);

  StreamingShardedWriteFactory<Object, Void, Object> factory =
      new StreamingShardedWriteFactory<>(p.getOptions());
  WriteFiles<Object, Void, Object> original = WriteFiles.to(new TestSink(tmpFolder.toString()));
  PCollection<Object> objs = (PCollection) p.apply(Create.empty(VoidCoder.of()));
  AppliedPTransform<PCollection<Object>, WriteFilesResult<Void>, WriteFiles<Object, Void, Object>>
      originalApplication =
          AppliedPTransform.of("writefiles", objs.expand(), Collections.emptyMap(), original, p);

  WriteFiles<Object, Void, Object> replacement =
      (WriteFiles<Object, Void, Object>)
          factory.getReplacementTransform(originalApplication).getTransform();
  assertThat(replacement, not(equalTo((Object) original)));
  assertThat(replacement.getNumShardsProvider().get(), equalTo(expectedNumShards));

  WriteFilesResult<Void> originalResult = objs.apply(original);
  WriteFilesResult<Void> replacementResult = objs.apply(replacement);
  Map<PValue, ReplacementOutput> res =
      factory.mapOutputs(originalResult.expand(), replacementResult);
  assertEquals(1, res.size());
  assertEquals(
      originalResult.getPerDestinationOutputFilenames(),
      res.get(replacementResult.getPerDestinationOutputFilenames()).getOriginal().getValue());
}
 
Example #15
Source File: BoundedReadEvaluatorFactoryTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void boundedSourceEvaluatorNoElementsClosesReader() throws Exception {
  TestSource<Long> source = new TestSource<>(BigEndianLongCoder.of());

  PCollection<Long> pcollection = p.apply(Read.from(source));
  AppliedPTransform<?, ?, ?> sourceTransform = DirectGraphs.getProducer(pcollection);

  UncommittedBundle<Long> output = bundleFactory.createBundle(pcollection);
  when(context.createBundle(pcollection)).thenReturn(output);

  TransformEvaluator<BoundedSourceShard<Long>> evaluator =
      factory.forApplication(
          sourceTransform, bundleFactory.createRootBundle().commit(Instant.now()));
  evaluator.processElement(WindowedValue.valueInGlobalWindow(BoundedSourceShard.of(source)));
  evaluator.finishBundle();
  CommittedBundle<Long> committed = output.commit(Instant.now());
  assertThat(committed.getElements(), emptyIterable());
  assertThat(TestSource.readerClosed, is(true));
}
 
Example #16
Source File: ParDoEvaluatorFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public <T> TransformEvaluator<T> forApplication(
    AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception {

  @SuppressWarnings({"unchecked", "rawtypes"})
  TransformEvaluator<T> evaluator =
      (TransformEvaluator<T>)
          createEvaluator(
              (AppliedPTransform) application,
              (PCollection<InputT>) inputBundle.getPCollection(),
              inputBundle.getKey(),
              ParDoTranslation.getSideInputs(application),
              (TupleTag<OutputT>) ParDoTranslation.getMainOutputTag(application),
              ParDoTranslation.getAdditionalOutputTags(application).getAll(),
              ParDoTranslation.getSchemaInformation(application),
              ParDoTranslation.getSideInputMapping(application));
  return evaluator;
}
 
Example #17
Source File: BoundedReadEvaluatorFactoryTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void boundedSourceEvaluatorClosesReader() throws Exception {
  TestSource<Long> source = new TestSource<>(BigEndianLongCoder.of(), 1L, 2L, 3L);
  PCollection<Long> pcollection = p.apply(Read.from(source));
  AppliedPTransform<?, ?, ?> sourceTransform = DirectGraphs.getProducer(pcollection);

  UncommittedBundle<Long> output = bundleFactory.createBundle(pcollection);
  when(context.createBundle(pcollection)).thenReturn(output);

  TransformEvaluator<BoundedSourceShard<Long>> evaluator =
      factory.forApplication(
          sourceTransform, bundleFactory.createRootBundle().commit(Instant.now()));
  evaluator.processElement(WindowedValue.valueInGlobalWindow(BoundedSourceShard.of(source)));
  evaluator.finishBundle();
  CommittedBundle<Long> committed = output.commit(Instant.now());
  assertThat(committed.getElements(), containsInAnyOrder(gw(2L), gw(3L), gw(1L)));
  assertThat(TestSource.readerClosed, is(true));
}
 
Example #18
Source File: PTransformMatchersTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void parDoWithFnTypeWithNoMatch() {
  DoFn<Object, Object> fn =
      new DoFn<Object, Object>() {
        @ProcessElement
        public void process(ProcessContext ctxt) {}
      };
  AppliedPTransform<?, ?, ?> parDoSingle = getAppliedTransform(ParDo.of(fn));
  AppliedPTransform<?, ?, ?> parDoMulti =
      getAppliedTransform(ParDo.of(fn).withOutputTags(new TupleTag<>(), TupleTagList.empty()));

  PTransformMatcher matcher = PTransformMatchers.parDoWithFnType(doFnWithState.getClass());
  assertThat(matcher.matches(parDoSingle), is(false));
  assertThat(matcher.matches(parDoMulti), is(false));
}
 
Example #19
Source File: CombineTranslationTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testToProtoWithoutSideInputs() throws Exception {
  PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3));
  CombineFnWithContext<Integer, int[], Integer> combineFn = new TestCombineFnWithContext();
  input.apply(Combine.globally(combineFn).withoutDefaults());
  final AtomicReference<AppliedPTransform<?, ?, Combine.Globally<?, ?>>> combine =
      new AtomicReference<>();
  pipeline.traverseTopologically(
      new PipelineVisitor.Defaults() {
        @Override
        public void leaveCompositeTransform(Node node) {
          if (node.getTransform() instanceof Combine.Globally) {
            checkState(combine.get() == null);
            combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline()));
          }
        }
      });
  checkState(combine.get() != null);
  assertEquals(combineFn, combine.get().getTransform().getFn());

  SdkComponents sdkComponents = SdkComponents.create();
  sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java"));
  CombinePayload combineProto =
      CombineTranslation.CombineGloballyPayloadTranslator.payloadForCombineGlobally(
          (AppliedPTransform) combine.get(), sdkComponents);
  RunnerApi.Components componentsProto = sdkComponents.toComponents();

  assertEquals(
      combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()),
      getAccumulatorCoder(combineProto, RehydratedComponents.forComponents(componentsProto)));
  assertEquals(
      combineFn,
      SerializableUtils.deserializeFromByteArray(
          combineProto.getCombineFn().getPayload().toByteArray(), "CombineFn"));
}
 
Example #20
Source File: DataflowMetricsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testIgnoreDistributionButGetCounterUpdates() throws IOException {
  JobMetrics jobMetrics = new JobMetrics();
  DataflowClient dataflowClient = mock(DataflowClient.class);
  when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
  DataflowPipelineJob job = mock(DataflowPipelineJob.class);
  DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
  when(options.isStreaming()).thenReturn(false);
  when(job.getDataflowOptions()).thenReturn(options);
  when(job.getState()).thenReturn(State.RUNNING);
  job.jobId = JOB_ID;

  AppliedPTransform<?, ?, ?> myStep = mock(AppliedPTransform.class);
  when(myStep.getFullName()).thenReturn("myStepName");
  job.transformStepNames = HashBiMap.create();
  job.transformStepNames.put(myStep, "s2");

  // The parser relies on the fact that one tentative and one committed metric update exist in
  // the job metrics results.
  jobMetrics.setMetrics(
      ImmutableList.of(
          makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false),
          makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, true),
          makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, false),
          makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, true)));

  DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
  MetricQueryResults result = dataflowMetrics.allMetrics();
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L)));
  assertThat(
      result.getCounters(),
      containsInAnyOrder(
          committedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L)));
}
 
Example #21
Source File: BatchStatefulParDoOverrides.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PTransformReplacement<PCollection<KV<K, InputT>>, PCollection<OutputT>>
    getReplacementTransform(
        AppliedPTransform<
                PCollection<KV<K, InputT>>,
                PCollection<OutputT>,
                SingleOutput<KV<K, InputT>, OutputT>>
            transform) {
  return PTransformReplacement.of(
      PTransformReplacements.getSingletonMainInput(transform),
      new StatefulSingleOutputParDo<>(transform.getTransform(), isFnApi));
}
 
Example #22
Source File: Utils.java    From beam with Apache License 2.0 5 votes vote down vote up
static boolean usesStateOrTimers(AppliedPTransform<?, ?, ?> appliedTransform) {
  try {
    return ParDoTranslation.usesStateOrTimers(appliedTransform);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #23
Source File: PrimitiveParDoSingleFactory.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PTransformReplacement<PCollection<? extends InputT>, PCollection<OutputT>>
    getReplacementTransform(
        AppliedPTransform<
                PCollection<? extends InputT>,
                PCollection<OutputT>,
                SingleOutput<InputT, OutputT>>
            transform) {
  return PTransformReplacement.of(
      PTransformReplacements.getSingletonMainInput(transform),
      new ParDoSingle<>(
          transform.getTransform(),
          Iterables.getOnlyElement(transform.getOutputs().keySet()),
          PTransformReplacements.getSingletonMainOutput(transform).getCoder()));
}
 
Example #24
Source File: TransformInputs.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Gets all inputs of the {@link AppliedPTransform} that are not returned by {@link
 * PTransform#getAdditionalInputs()}.
 */
public static Collection<PValue> nonAdditionalInputs(AppliedPTransform<?, ?, ?> application) {
  ImmutableList.Builder<PValue> mainInputs = ImmutableList.builder();
  PTransform<?, ?> transform = application.getTransform();
  for (Map.Entry<TupleTag<?>, PValue> input : application.getInputs().entrySet()) {
    if (!transform.getAdditionalInputs().containsKey(input.getKey())) {
      mainInputs.add(input.getValue());
    }
  }
  checkArgument(
      !mainInputs.build().isEmpty() || application.getInputs().isEmpty(),
      "Expected at least one main input if any inputs exist");
  return mainInputs.build();
}
 
Example #25
Source File: Utils.java    From beam with Apache License 2.0 5 votes vote down vote up
static List<PCollectionView<?>> getSideInputs(AppliedPTransform<?, ?, ?> appliedTransform) {
  PTransform<?, ?> transform = appliedTransform.getTransform();
  if (transform instanceof ParDo.MultiOutput) {
    ParDo.MultiOutput multiParDo = (ParDo.MultiOutput) transform;
    return (List) multiParDo.getSideInputs().values().stream().collect(Collectors.toList());
  } else if (transform instanceof ParDo.SingleOutput) {
    ParDo.SingleOutput singleParDo = (ParDo.SingleOutput) transform;
    return (List) singleParDo.getSideInputs().values().stream().collect(Collectors.toList());
  }
  return Collections.emptyList();
}
 
Example #26
Source File: DataflowRunner.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
    AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) {
  Create.Values<T> original = transform.getTransform();
  PCollection<T> output =
      (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
  return PTransformReplacement.of(
      transform.getPipeline().begin(), new StreamingFnApiCreate<>(original, output));
}
 
Example #27
Source File: CommittedResultTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void getUncommittedElementsNull() {
  CommittedResult<AppliedPTransform<?, ?, ?>> result =
      CommittedResult.create(
          StepTransformResult.withoutHold(transform).build(),
          Optional.empty(),
          Collections.emptyList(),
          EnumSet.noneOf(OutputType.class));

  assertThat(result.getUnprocessedInputs(), Matchers.equalTo(Optional.empty()));
}
 
Example #28
Source File: PipelineTranslationTest.java    From beam with Apache License 2.0 5 votes vote down vote up
private static Optional<CombinePayload> getCombinePayload(
    AppliedPTransform<?, ?, ?> transform, SdkComponents components) throws IOException {
  RunnerApi.PTransform proto =
      PTransformTranslation.toProto(transform, Collections.emptyList(), components);

  // Even if the proto has no spec, calling getSpec still returns a blank spec, which we want to
  // avoid. It should be clear to the caller whether or not there was a spec in the transform.
  if (proto.hasSpec()) {
    return Optional.of(CombinePayload.parseFrom(proto.getSpec().getPayload()));
  } else {
    return Optional.empty();
  }
}
 
Example #29
Source File: DataflowRunner.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PTransformReplacement<PCollection<PubsubMessage>, PDone> getReplacementTransform(
    AppliedPTransform<PCollection<PubsubMessage>, PDone, PubsubUnboundedSink> transform) {
  return PTransformReplacement.of(
      PTransformReplacements.getSingletonMainInput(transform),
      new StreamingPubsubIOWrite(runner, transform.getTransform()));
}
 
Example #30
Source File: PTransformMatchers.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * A {@link PTransformMatcher} that matches a {@link ParDo.MultiOutput} containing a {@link DoFn}
 * that uses state or timers, as specified by {@link DoFnSignature#usesState()} and {@link
 * DoFnSignature#usesTimers()}.
 */
public static PTransformMatcher stateOrTimerParDoMulti() {
  return new PTransformMatcher() {
    @Override
    public boolean matches(AppliedPTransform<?, ?, ?> application) {
      PTransform<?, ?> transform = application.getTransform();
      if (transform instanceof ParDo.MultiOutput) {
        DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn();
        DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn);
        return signature.usesState() || signature.usesTimers();
      }
      return false;
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper("StateOrTimerParDoMultiMatcher").toString();
    }
  };
}