org.apache.beam.sdk.coders.IterableCoder Java Examples

The following examples show how to use org.apache.beam.sdk.coders.IterableCoder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AggregatorCombiner.java    From beam with Apache License 2.0 7 votes vote down vote up
public AggregatorCombiner(
    Combine.CombineFn<InputT, AccumT, OutputT> combineFn,
    WindowingStrategy<?, ?> windowingStrategy,
    Coder<AccumT> accumulatorCoder,
    Coder<OutputT> outputCoder) {
  this.combineFn = combineFn;
  this.windowingStrategy = (WindowingStrategy<InputT, W>) windowingStrategy;
  this.timestampCombiner = windowingStrategy.getTimestampCombiner();
  this.accumulatorCoder =
      IterableCoder.of(
          WindowedValue.FullWindowedValueCoder.of(
              accumulatorCoder, windowingStrategy.getWindowFn().windowCoder()));
  this.outputCoder =
      IterableCoder.of(
          WindowedValue.FullWindowedValueCoder.of(
              outputCoder, windowingStrategy.getWindowFn().windowCoder()));
}
 
Example #2
Source File: CombineValuesFnFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<InputT>>, KV<K, AccumT>> doFn =
      new AddInputsDoFn<>(combineFnRunner, sideInputReader);

  Coder<KV<K, Iterable<InputT>>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getKvCoder().getValueCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 
Example #3
Source File: ValueAndCoderLazySerializableTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void serializableAccumulatorSerializationTest()
    throws IOException, ClassNotFoundException {
  Iterable<WindowedValue<Integer>> accumulatedValue =
      Arrays.asList(winVal(0), winVal(1), winVal(3), winVal(4));

  final WindowedValue.FullWindowedValueCoder<Integer> wvaCoder =
      WindowedValue.FullWindowedValueCoder.of(
          BigEndianIntegerCoder.of(), GlobalWindow.Coder.INSTANCE);

  final IterableCoder<WindowedValue<Integer>> iterAccumCoder = IterableCoder.of(wvaCoder);

  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> accUnderTest =
      ValueAndCoderLazySerializable.of(accumulatedValue, iterAccumCoder);

  ByteArrayOutputStream inMemOut = new ByteArrayOutputStream();
  ObjectOutputStream oos = new ObjectOutputStream(inMemOut);
  oos.writeObject(accUnderTest);

  ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(inMemOut.toByteArray()));

  @SuppressWarnings("unchecked")
  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> materialized =
      (ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>>) ois.readObject();
  assertEquals(accumulatedValue, materialized.getOrDecode(iterAccumCoder));
}
 
Example #4
Source File: BigtableIOTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that when writing to a non-existent table, the write fails. */
@Test
public void testWritingFailsTableDoesNotExist() throws Exception {
  final String table = "TEST-TABLE";

  PCollection<KV<ByteString, Iterable<Mutation>>> emptyInput =
      p.apply(
          Create.empty(
              KvCoder.of(ByteStringCoder.of(), IterableCoder.of(ProtoCoder.of(Mutation.class)))));

  // Exception will be thrown by write.validate() when writeToDynamic is applied.
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(String.format("Table %s does not exist", table));

  emptyInput.apply("write", defaultWrite.withTableId(table));
  p.run();
}
 
Example #5
Source File: FlattenTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(ValidatesRunner.class)
public void testFlattenWithDifferentInputAndOutputCoders2() {
  // This test exists to prevent a regression in Dataflow. It tests a
  // GroupByKey followed by a Flatten with an SDK-specific output coder.
  PCollection<KV<String, Iterable<String>>> flattenInput =
      p.apply(Create.of(LINES))
          .apply(WithKeys.of("a"))
          .setCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
          .apply(GroupByKey.create());
  PCollection<String> output =
      PCollectionList.of(flattenInput)
          .apply(Flatten.pCollections())
          .setCoder(SerializableCoder.of(new TypeDescriptor<KV<String, Iterable<String>>>() {}))
          .apply(Values.create())
          .setCoder(IterableCoder.of(StringUtf8Coder.of()))
          .apply(
              FlatMapElements.into(TypeDescriptors.strings())
                  .via((Iterable<String> values) -> values));
  PAssert.that(output).containsInAnyOrder(LINES);
  p.run();
}
 
Example #6
Source File: BatchViewOverrides.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Transforms the input {@link PCollection} into a singleton {@link Map} per window. */
private <W extends BoundedWindow> PCollection<?> applyForSingletonFallback(
    PCollection<KV<K, V>> input) {
  @SuppressWarnings("unchecked")
  Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder();

  @SuppressWarnings({"rawtypes", "unchecked"})
  KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();

  @SuppressWarnings({"unchecked", "rawtypes"})
  Coder<Function<Iterable<WindowedValue<V>>, Iterable<V>>> transformCoder =
      (Coder) SerializableCoder.of(IterableWithWindowedValuesToIterable.class);

  Coder<TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>> finalValueCoder =
      TransformedMapCoder.of(
          transformCoder,
          MapCoder.of(
              inputCoder.getKeyCoder(),
              IterableCoder.of(
                  FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder))));

  return BatchViewAsSingleton.applyForSingleton(
      runner, input, new ToMultimapDoFn<>(windowCoder), finalValueCoder, view);
}
 
Example #7
Source File: ReifyAsIterable.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PCollection<Iterable<T>> expand(PCollection<T> input) {
  final PCollectionView<Iterable<T>> view = input.apply(View.asIterable());
  return input
      .getPipeline()
      .apply(Create.of((Void) null).withCoder(VoidCoder.of()))
      .apply(
          ParDo.of(
                  new DoFn<Void, Iterable<T>>() {
                    @ProcessElement
                    public void processElement(ProcessContext c) {
                      c.output(c.sideInput(view));
                    }
                  })
              .withSideInputs(view))
      .setCoder(IterableCoder.of(input.getCoder()));
}
 
Example #8
Source File: SparkGroupAlsoByWindowViaWindowSet.java    From beam with Apache License 2.0 6 votes vote down vote up
UpdateStateByKeyFunction(
    final List<Integer> sourceIds,
    final WindowingStrategy<?, W> windowingStrategy,
    final FullWindowedValueCoder<InputT> wvCoder,
    final Coder<K> keyCoder,
    final SerializablePipelineOptions options,
    final String logPrefix) {
  this.wvCoder = wvCoder;
  this.keyCoder = keyCoder;
  this.sourceIds = sourceIds;
  this.timerDataCoder = timerDataCoderOf(windowingStrategy);
  this.windowingStrategy = windowingStrategy;
  this.options = options;
  this.itrWvCoder = IterableCoder.of(wvCoder);
  this.logPrefix = logPrefix;
  this.wvKvIterCoder =
      windowedValueKeyValueCoderOf(
          keyCoder,
          wvCoder.getValueCoder(),
          ((FullWindowedValueCoder<InputT>) wvCoder).getWindowCoder());
}
 
Example #9
Source File: ToStringTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testToStringIterableWithDelimiter() {
  ArrayList<Iterable<String>> iterables = new ArrayList<>();
  iterables.add(Arrays.asList(new String[] {"one", "two", "three"}));
  iterables.add(Arrays.asList(new String[] {"four", "five", "six"}));

  ArrayList<String> expected = new ArrayList<>();
  expected.add("one\ttwo\tthree");
  expected.add("four\tfive\tsix");

  PCollection<Iterable<String>> input =
      p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of())));
  PCollection<String> output = input.apply(ToString.iterables("\t"));
  PAssert.that(output).containsInAnyOrder(expected);
  p.run();
}
 
Example #10
Source File: ToStringTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testToStringIterable() {
  ArrayList<Iterable<String>> iterables = new ArrayList<>();
  iterables.add(Arrays.asList(new String[] {"one", "two", "three"}));
  iterables.add(Arrays.asList(new String[] {"four", "five", "six"}));

  ArrayList<String> expected = new ArrayList<>();
  expected.add("one,two,three");
  expected.add("four,five,six");

  PCollection<Iterable<String>> input =
      p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of())));
  PCollection<String> output = input.apply(ToString.iterables());
  PAssert.that(output).containsInAnyOrder(expected);
  p.run();
}
 
Example #11
Source File: CombineValuesFnFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<InputT>>, KV<K, OutputT>> doFn =
      new CombineValuesDoFn<>(combineFnRunner, sideInputReader);

  Coder<KV<K, Iterable<InputT>>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getKvCoder().getValueCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 
Example #12
Source File: PAssert.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PCollectionView<ActualT> expand(PBegin input) {
  final Coder<T> coder = actual.getCoder();
  return actual
      .apply("FilterActuals", rewindowActuals.prepareActuals())
      .apply("GatherPanes", GatherAllPanes.globally())
      .apply("ExtractPane", MapElements.via(extractPane))
      .setCoder(IterableCoder.of(actual.getCoder()))
      .apply(Flatten.iterables())
      .apply("RewindowActuals", rewindowActuals.windowActuals())
      .apply(
          ParDo.of(
              new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext context) throws CoderException {
                  context.output(CoderUtils.clone(coder, context.element()));
                }
              }))
      .apply(actualView);
}
 
Example #13
Source File: CombineValuesFnFactory.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<AccumT>>, KV<K, AccumT>> doFn =
      new MergeAccumulatorsDoFn<>(combineFnRunner, sideInputReader);

  KvCoder<K, Iterable<AccumT>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getAccumulatorCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 
Example #14
Source File: CombineTranslation.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<
            PCollection<KV<K, Iterable<InputT>>>, ?, Combine.GroupedValues<K, InputT, ?>>
        transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<KV<K, Iterable<InputT>>> mainInput =
        (PCollection<KV<K, Iterable<InputT>>>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    KvCoder<K, Iterable<InputT>> kvCoder = (KvCoder<K, Iterable<InputT>>) mainInput.getCoder();
    IterableCoder<InputT> iterCoder = (IterableCoder<InputT>) kvCoder.getValueCoder();
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(), iterCoder.getElemCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 
Example #15
Source File: SdkComponentsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void registerCoder() throws IOException {
  Coder<?> coder =
      KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of())));
  String id = components.registerCoder(coder);
  assertThat(components.registerCoder(coder), equalTo(id));
  assertThat(id, not(isEmptyOrNullString()));
  Coder<?> equalCoder =
      KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of())));
  assertThat(components.registerCoder(equalCoder), equalTo(id));
  Coder<?> otherCoder = VarLongCoder.of();
  assertThat(components.registerCoder(otherCoder), not(equalTo(id)));

  components.toComponents().getCodersOrThrow(id);
  components.toComponents().getCodersOrThrow(components.registerCoder(otherCoder));
}
 
Example #16
Source File: Combine.java    From beam with Apache License 2.0 6 votes vote down vote up
private KvCoder<K, InputT> getKvCoder(
    Coder<? extends KV<K, ? extends Iterable<InputT>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("Combine.GroupedValues requires its input to use KvCoder");
  }
  @SuppressWarnings({"unchecked", "rawtypes"})
  KvCoder<K, ? extends Iterable<InputT>> kvCoder = (KvCoder) inputCoder;
  Coder<K> keyCoder = kvCoder.getKeyCoder();
  Coder<? extends Iterable<InputT>> kvValueCoder = kvCoder.getValueCoder();
  if (!(kvValueCoder instanceof IterableCoder)) {
    throw new IllegalStateException(
        "Combine.GroupedValues requires its input values to use " + "IterableCoder");
  }
  @SuppressWarnings("unchecked")
  IterableCoder<InputT> inputValuesCoder = (IterableCoder<InputT>) kvValueCoder;
  Coder<InputT> inputValueCoder = inputValuesCoder.getElemCoder();
  return KvCoder.of(keyCoder, inputValueCoder);
}
 
Example #17
Source File: ModelCodersTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void windowedValueCoderComponentsToConstructor() throws IOException {
  FullWindowedValueCoder<Iterable<KV<String, Integer>>> javaCoder =
      FullWindowedValueCoder.of(
          IterableCoder.of(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())),
          IntervalWindowCoder.of());

  MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder);
  WindowedValueCoderComponents windowedValueCoderComponents =
      ModelCoders.getWindowedValueCoderComponents(coderAndComponents.getCoder());

  Coder windowedCoder =
      ModelCoders.windowedValueCoder(
          windowedValueCoderComponents.elementCoderId(),
          windowedValueCoderComponents.windowCoderId());

  assertThat(windowedCoder, equalTo(coderAndComponents.getCoder()));
}
 
Example #18
Source File: GroupByKeyViaGroupByKeyOnly.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PCollection<KV<K, Iterable<V>>> expand(
    PCollection<KV<K, Iterable<WindowedValue<V>>>> input) {
  @SuppressWarnings("unchecked")
  KvCoder<K, Iterable<WindowedValue<V>>> inputKvCoder =
      (KvCoder<K, Iterable<WindowedValue<V>>>) input.getCoder();

  Coder<K> keyCoder = inputKvCoder.getKeyCoder();
  Coder<Iterable<WindowedValue<V>>> inputValueCoder = inputKvCoder.getValueCoder();

  IterableCoder<WindowedValue<V>> inputIterableValueCoder =
      (IterableCoder<WindowedValue<V>>) inputValueCoder;
  Coder<WindowedValue<V>> inputIterableElementCoder = inputIterableValueCoder.getElemCoder();
  WindowedValueCoder<V> inputIterableWindowedValueCoder =
      (WindowedValueCoder<V>) inputIterableElementCoder;

  Coder<V> inputIterableElementValueCoder = inputIterableWindowedValueCoder.getValueCoder();
  Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder);
  Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder);

  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder);
}
 
Example #19
Source File: SortValues.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Retrieves the {@link Coder} for the secondary key-value pairs. */
@SuppressWarnings("unchecked")
private static <PrimaryKeyT, SecondaryKeyT, ValueT>
    KvCoder<SecondaryKeyT, ValueT> getSecondaryKeyValueCoder(
        Coder<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("SortValues requires its input to use KvCoder");
  }
  @SuppressWarnings("unchecked")
  KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>> kvCoder =
      (KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>) inputCoder;

  if (!(kvCoder.getValueCoder() instanceof IterableCoder)) {
    throw new IllegalStateException(
        "SortValues requires the values be encoded with IterableCoder");
  }
  IterableCoder<KV<SecondaryKeyT, ValueT>> iterableCoder =
      (IterableCoder<KV<SecondaryKeyT, ValueT>>) (kvCoder.getValueCoder());

  if (!(iterableCoder.getElemCoder() instanceof KvCoder)) {
    throw new IllegalStateException(
        "SortValues requires the secondary key-value pairs to use KvCoder");
  }
  return (KvCoder<SecondaryKeyT, ValueT>) (iterableCoder.getElemCoder());
}
 
Example #20
Source File: GroupingTablesTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testBufferingGroupingTable() throws Exception {
  GroupingTableBase<String, String, List<String>> table =
      (GroupingTableBase<String, String, List<String>>)
          GroupingTables.buffering(
              new IdentityGroupingKeyCreator(), new KvPairInfo(),
              new StringPowerSizeEstimator(), new StringPowerSizeEstimator());
  table.setMaxSize(1000);
  TestOutputReceiver receiver =
      new TestOutputReceiver(
          KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(StringUtf8Coder.of())),
          NameContextsForTests.nameContextForTest());

  table.put("A", "a", receiver);
  table.put("B", "b1", receiver);
  table.put("B", "b2", receiver);
  table.put("C", "c", receiver);
  assertThat(receiver.outputElems, empty());

  table.put("C", "cccc", receiver);
  assertThat(receiver.outputElems, hasItem((Object) KV.of("C", Arrays.asList("c", "cccc"))));

  table.put("DDDD", "d", receiver);
  assertThat(receiver.outputElems, hasItem((Object) KV.of("DDDD", Arrays.asList("d"))));

  table.flush(receiver);
  assertThat(
      receiver.outputElems,
      IsIterableContainingInAnyOrder.<Object>containsInAnyOrder(
          KV.of("A", Arrays.asList("a")),
          KV.of("B", Arrays.asList("b1", "b2")),
          KV.of("C", Arrays.asList("c", "cccc")),
          KV.of("DDDD", Arrays.asList("d"))));
}
 
Example #21
Source File: IntrinsicMapTaskExecutorFactoryTest.java    From beam with Apache License 2.0 5 votes vote down vote up
static ParallelInstruction createPartialGroupByKeyInstruction(
    int producerIndex, int producerOutputNum) {
  InstructionInput cloudInput = new InstructionInput();
  cloudInput.setProducerInstructionIndex(producerIndex);
  cloudInput.setOutputNum(producerOutputNum);

  PartialGroupByKeyInstruction pgbkInstruction = new PartialGroupByKeyInstruction();
  pgbkInstruction.setInput(cloudInput);
  pgbkInstruction.setInputElementCodec(
      CloudObjects.asCloudObject(
          FullWindowedValueCoder.of(
              KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()),
              IntervalWindowCoder.of()),
          /*sdkComponents=*/ null));

  InstructionOutput output = new InstructionOutput();
  output.setName("pgbk_output_name");
  output.setCodec(
      CloudObjects.asCloudObject(
          KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(BigEndianIntegerCoder.of())),
          /*sdkComponents=*/ null));
  output.setOriginalName("originalName");
  output.setSystemName("systemName");

  ParallelInstruction instruction = new ParallelInstruction();
  instruction.setOriginalName("pgbk_original_name");
  instruction.setSystemName("pgbk_system_name");
  instruction.setPartialGroupByKey(pgbkInstruction);
  instruction.setOutputs(Arrays.asList(output));

  return instruction;
}
 
Example #22
Source File: RemoteInputDestinationTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testConstruction() {
  String transformId = "my_target_pt";

  KvCoder<byte[], Iterable<Long>> coder =
      KvCoder.of(LengthPrefixCoder.of(ByteArrayCoder.of()), IterableCoder.of(VarLongCoder.of()));
  RemoteInputDestination<KV<byte[], Iterable<Long>>> destination =
      RemoteInputDestination.of(coder, transformId);

  assertThat(destination.getCoder(), equalTo(coder));
  assertThat(destination.getPTransformId(), equalTo(transformId));
}
 
Example #23
Source File: DirectGroupByKey.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) {
  KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder());
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      outputWindowingStrategy,
      input.isBounded(),
      KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder())));
}
 
Example #24
Source File: GroupByKeyViaGroupByKeyOnly.java    From beam with Apache License 2.0 5 votes vote down vote up
public Coder<V> getValueCoder(Coder<KV<K, Iterable<WindowedValue<V>>>> inputCoder) {
  // Coder<Iterable<...>> --> IterableCoder<...>
  Coder<Iterable<WindowedValue<V>>> iterableWindowedValueCoder =
      getKvCoder(inputCoder).getValueCoder();
  checkArgument(
      iterableWindowedValueCoder instanceof IterableCoder,
      "%s requires a %s<..., %s> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      iterableWindowedValueCoder);
  IterableCoder<WindowedValue<V>> iterableCoder =
      (IterableCoder<WindowedValue<V>>) iterableWindowedValueCoder;

  // Coder<WindowedValue<...>> --> WindowedValueCoder<...>
  Coder<WindowedValue<V>> iterableElementCoder = iterableCoder.getElemCoder();
  checkArgument(
      iterableElementCoder instanceof WindowedValueCoder,
      "%s requires a %s<..., %s<%s>> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      WindowedValueCoder.class.getSimpleName(),
      iterableElementCoder);
  WindowedValueCoder<V> windowedValueCoder = (WindowedValueCoder<V>) iterableElementCoder;

  return windowedValueCoder.getValueCoder();
}
 
Example #25
Source File: TransformTranslator.java    From beam with Apache License 2.0 5 votes vote down vote up
private static <ReadT, WriteT>
    TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>> createPCollView() {
  return new TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>() {
    @Override
    public void evaluate(
        View.CreatePCollectionView<ReadT, WriteT> transform, EvaluationContext context) {
      Iterable<? extends WindowedValue<?>> iter =
          context.getWindowedValues(context.getInput(transform));
      PCollectionView<WriteT> output = transform.getView();
      Coder<Iterable<WindowedValue<?>>> coderInternal =
          (Coder)
              IterableCoder.of(
                  WindowedValue.getFullCoder(
                      output.getCoderInternal(),
                      output.getWindowingStrategyInternal().getWindowFn().windowCoder()));

      @SuppressWarnings("unchecked")
      Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;

      context.putPView(output, iterCast, coderInternal);
    }

    @Override
    public String toNativeString() {
      return "<createPCollectionView>";
    }
  };
}
 
Example #26
Source File: SparkGroupAlsoByWindowViaWindowSet.java    From beam with Apache License 2.0 5 votes vote down vote up
private static <K, InputT>
    FullWindowedValueCoder<KV<K, Iterable<InputT>>> windowedValueKeyValueCoderOf(
        final Coder<K> keyCoder,
        final Coder<InputT> iCoder,
        final Coder<? extends BoundedWindow> wCoder) {
  return FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder);
}
 
Example #27
Source File: GroupByKeyTranslatorBatch.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public void translateTransform(
    PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> transform,
    TranslationContext context) {

  @SuppressWarnings("unchecked")
  final PCollection<KV<K, V>> inputPCollection = (PCollection<KV<K, V>>) context.getInput();
  Dataset<WindowedValue<KV<K, V>>> input = context.getDataset(inputPCollection);
  WindowingStrategy<?, ?> windowingStrategy = inputPCollection.getWindowingStrategy();
  KvCoder<K, V> kvCoder = (KvCoder<K, V>) inputPCollection.getCoder();
  Coder<V> valueCoder = kvCoder.getValueCoder();

  // group by key only
  Coder<K> keyCoder = kvCoder.getKeyCoder();
  KeyValueGroupedDataset<K, WindowedValue<KV<K, V>>> groupByKeyOnly =
      input.groupByKey(KVHelpers.extractKey(), EncoderHelpers.fromBeamCoder(keyCoder));

  // group also by windows
  WindowedValue.FullWindowedValueCoder<KV<K, Iterable<V>>> outputCoder =
      WindowedValue.FullWindowedValueCoder.of(
          KvCoder.of(keyCoder, IterableCoder.of(valueCoder)),
          windowingStrategy.getWindowFn().windowCoder());
  Dataset<WindowedValue<KV<K, Iterable<V>>>> output =
      groupByKeyOnly.flatMapGroups(
          new GroupAlsoByWindowViaOutputBufferFn<>(
              windowingStrategy,
              new InMemoryStateInternalsFactory<>(),
              SystemReduceFn.buffering(valueCoder),
              context.getSerializableOptions()),
          EncoderHelpers.fromBeamCoder(outputCoder));

  context.putDataset(context.getOutput(), output);
}
 
Example #28
Source File: ReduceFnTester.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link
 * TriggerStateMachine}, for mocking the interactions between {@link ReduceFnRunner} and the
 * {@link TriggerStateMachine}.
 *
 * <p>Ignores the {@link Trigger} on the {@link WindowingStrategy}.
 */
public static <W extends BoundedWindow>
    ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining(
        WindowingStrategy<?, W> windowingStrategy, TriggerStateMachine triggerStateMachine)
        throws Exception {
  return new ReduceFnTester<>(
      windowingStrategy,
      triggerStateMachine,
      SystemReduceFn.buffering(VarIntCoder.of()),
      IterableCoder.of(VarIntCoder.of()),
      PipelineOptionsFactory.create(),
      NullSideInputReader.empty());
}
 
Example #29
Source File: ReduceFnTester.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, creating a {@link
 * TriggerStateMachine} from its {@link Trigger}.
 */
public static <W extends BoundedWindow>
    ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining(
        WindowingStrategy<?, W> windowingStrategy) throws Exception {
  return new ReduceFnTester<>(
      windowingStrategy,
      TriggerStateMachines.stateMachineForTrigger(
          TriggerTranslation.toProto(windowingStrategy.getTrigger())),
      SystemReduceFn.buffering(VarIntCoder.of()),
      IterableCoder.of(VarIntCoder.of()),
      PipelineOptionsFactory.create(),
      NullSideInputReader.empty());
}
 
Example #30
Source File: KeyedWorkItemCoder.java    From beam with Apache License 2.0 5 votes vote down vote up
private KeyedWorkItemCoder(
    Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) {
  this.keyCoder = keyCoder;
  this.elemCoder = elemCoder;
  this.windowCoder = windowCoder;
  this.timersCoder = IterableCoder.of(TimerDataCoderV2.of(windowCoder));
  this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder));
}