org.apache.beam.sdk.coders.IterableCoder Java Examples

public AggregatorCombiner(
    Combine.CombineFn<InputT, AccumT, OutputT> combineFn,
    WindowingStrategy<?, ?> windowingStrategy,
    Coder<AccumT> accumulatorCoder,
    Coder<OutputT> outputCoder) {
  this.combineFn = combineFn;
  this.windowingStrategy = (WindowingStrategy<InputT, W>) windowingStrategy;
  this.timestampCombiner = windowingStrategy.getTimestampCombiner();
  this.accumulatorCoder =
      IterableCoder.of(
          WindowedValue.FullWindowedValueCoder.of(
              accumulatorCoder, windowingStrategy.getWindowFn().windowCoder()));
  this.outputCoder =
      IterableCoder.of(
          WindowedValue.FullWindowedValueCoder.of(
              outputCoder, windowingStrategy.getWindowFn().windowCoder()));
}
 

private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<InputT>>, KV<K, AccumT>> doFn =
      new AddInputsDoFn<>(combineFnRunner, sideInputReader);

  Coder<KV<K, Iterable<InputT>>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getKvCoder().getValueCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 

@Test
public void serializableAccumulatorSerializationTest()
    throws IOException, ClassNotFoundException {
  Iterable<WindowedValue<Integer>> accumulatedValue =
      Arrays.asList(winVal(0), winVal(1), winVal(3), winVal(4));

  final WindowedValue.FullWindowedValueCoder<Integer> wvaCoder =
      WindowedValue.FullWindowedValueCoder.of(
          BigEndianIntegerCoder.of(), GlobalWindow.Coder.INSTANCE);

  final IterableCoder<WindowedValue<Integer>> iterAccumCoder = IterableCoder.of(wvaCoder);

  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> accUnderTest =
      ValueAndCoderLazySerializable.of(accumulatedValue, iterAccumCoder);

  ByteArrayOutputStream inMemOut = new ByteArrayOutputStream();
  ObjectOutputStream oos = new ObjectOutputStream(inMemOut);
  oos.writeObject(accUnderTest);

  ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(inMemOut.toByteArray()));

  @SuppressWarnings("unchecked")
  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> materialized =
      (ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>>) ois.readObject();
  assertEquals(accumulatedValue, materialized.getOrDecode(iterAccumCoder));
}
 

/** Tests that when writing to a non-existent table, the write fails. */
@Test
public void testWritingFailsTableDoesNotExist() throws Exception {
  final String table = "TEST-TABLE";

  PCollection<KV<ByteString, Iterable<Mutation>>> emptyInput =
      p.apply(
          Create.empty(
              KvCoder.of(ByteStringCoder.of(), IterableCoder.of(ProtoCoder.of(Mutation.class)))));

  // Exception will be thrown by write.validate() when writeToDynamic is applied.
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(String.format("Table %s does not exist", table));

  emptyInput.apply("write", defaultWrite.withTableId(table));
  p.run();
}
 

@Test
@Category(ValidatesRunner.class)
public void testFlattenWithDifferentInputAndOutputCoders2() {
  // This test exists to prevent a regression in Dataflow. It tests a
  // GroupByKey followed by a Flatten with an SDK-specific output coder.
  PCollection<KV<String, Iterable<String>>> flattenInput =
      p.apply(Create.of(LINES))
          .apply(WithKeys.of("a"))
          .setCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
          .apply(GroupByKey.create());
  PCollection<String> output =
      PCollectionList.of(flattenInput)
          .apply(Flatten.pCollections())
          .setCoder(SerializableCoder.of(new TypeDescriptor<KV<String, Iterable<String>>>() {}))
          .apply(Values.create())
          .setCoder(IterableCoder.of(StringUtf8Coder.of()))
          .apply(
              FlatMapElements.into(TypeDescriptors.strings())
                  .via((Iterable<String> values) -> values));
  PAssert.that(output).containsInAnyOrder(LINES);
  p.run();
}
 

/** Transforms the input {@link PCollection} into a singleton {@link Map} per window. */
private <W extends BoundedWindow> PCollection<?> applyForSingletonFallback(
    PCollection<KV<K, V>> input) {
  @SuppressWarnings("unchecked")
  Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder();

  @SuppressWarnings({"rawtypes", "unchecked"})
  KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();

  @SuppressWarnings({"unchecked", "rawtypes"})
  Coder<Function<Iterable<WindowedValue<V>>, Iterable<V>>> transformCoder =
      (Coder) SerializableCoder.of(IterableWithWindowedValuesToIterable.class);

  Coder<TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>> finalValueCoder =
      TransformedMapCoder.of(
          transformCoder,
          MapCoder.of(
              inputCoder.getKeyCoder(),
              IterableCoder.of(
                  FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder))));

  return BatchViewAsSingleton.applyForSingleton(
      runner, input, new ToMultimapDoFn<>(windowCoder), finalValueCoder, view);
}
 

@Override
public PCollection<Iterable<T>> expand(PCollection<T> input) {
  final PCollectionView<Iterable<T>> view = input.apply(View.asIterable());
  return input
      .getPipeline()
      .apply(Create.of((Void) null).withCoder(VoidCoder.of()))
      .apply(
          ParDo.of(
                  new DoFn<Void, Iterable<T>>() {
                    @ProcessElement
                    public void processElement(ProcessContext c) {
                      c.output(c.sideInput(view));
                    }
                  })
              .withSideInputs(view))
      .setCoder(IterableCoder.of(input.getCoder()));
}
 

UpdateStateByKeyFunction(
    final List<Integer> sourceIds,
    final WindowingStrategy<?, W> windowingStrategy,
    final FullWindowedValueCoder<InputT> wvCoder,
    final Coder<K> keyCoder,
    final SerializablePipelineOptions options,
    final String logPrefix) {
  this.wvCoder = wvCoder;
  this.keyCoder = keyCoder;
  this.sourceIds = sourceIds;
  this.timerDataCoder = timerDataCoderOf(windowingStrategy);
  this.windowingStrategy = windowingStrategy;
  this.options = options;
  this.itrWvCoder = IterableCoder.of(wvCoder);
  this.logPrefix = logPrefix;
  this.wvKvIterCoder =
      windowedValueKeyValueCoderOf(
          keyCoder,
          wvCoder.getValueCoder(),
          ((FullWindowedValueCoder<InputT>) wvCoder).getWindowCoder());
}
 

@Test
@Category(NeedsRunner.class)
public void testToStringIterableWithDelimiter() {
  ArrayList<Iterable<String>> iterables = new ArrayList<>();
  iterables.add(Arrays.asList(new String[] {"one", "two", "three"}));
  iterables.add(Arrays.asList(new String[] {"four", "five", "six"}));

  ArrayList<String> expected = new ArrayList<>();
  expected.add("one\ttwo\tthree");
  expected.add("four\tfive\tsix");

  PCollection<Iterable<String>> input =
      p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of())));
  PCollection<String> output = input.apply(ToString.iterables("\t"));
  PAssert.that(output).containsInAnyOrder(expected);
  p.run();
}
 

@Test
@Category(NeedsRunner.class)
public void testToStringIterable() {
  ArrayList<Iterable<String>> iterables = new ArrayList<>();
  iterables.add(Arrays.asList(new String[] {"one", "two", "three"}));
  iterables.add(Arrays.asList(new String[] {"four", "five", "six"}));

  ArrayList<String> expected = new ArrayList<>();
  expected.add("one,two,three");
  expected.add("four,five,six");

  PCollection<Iterable<String>> input =
      p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of())));
  PCollection<String> output = input.apply(ToString.iterables());
  PAssert.that(output).containsInAnyOrder(expected);
  p.run();
}
 

private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<InputT>>, KV<K, OutputT>> doFn =
      new CombineValuesDoFn<>(combineFnRunner, sideInputReader);

  Coder<KV<K, Iterable<InputT>>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getKvCoder().getValueCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 

@Override
public PCollectionView<ActualT> expand(PBegin input) {
  final Coder<T> coder = actual.getCoder();
  return actual
      .apply("FilterActuals", rewindowActuals.prepareActuals())
      .apply("GatherPanes", GatherAllPanes.globally())
      .apply("ExtractPane", MapElements.via(extractPane))
      .setCoder(IterableCoder.of(actual.getCoder()))
      .apply(Flatten.iterables())
      .apply("RewindowActuals", rewindowActuals.windowActuals())
      .apply(
          ParDo.of(
              new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext context) throws CoderException {
                  context.output(CoderUtils.clone(coder, context.element()));
                }
              }))
      .apply(actualView);
}
 

private static <K, InputT, AccumT, OutputT> DoFnInfo<?, ?> createDoFnInfo(
    AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn, SideInputReader sideInputReader) {
  GlobalCombineFnRunner<InputT, AccumT, OutputT> combineFnRunner =
      GlobalCombineFnRunners.create(combineFn.getFn());
  DoFn<KV<K, Iterable<AccumT>>, KV<K, AccumT>> doFn =
      new MergeAccumulatorsDoFn<>(combineFnRunner, sideInputReader);

  KvCoder<K, Iterable<AccumT>> inputCoder = null;
  if (combineFn.getKvCoder() != null) {
    inputCoder =
        KvCoder.of(
            combineFn.getKvCoder().getKeyCoder(),
            IterableCoder.of(combineFn.getAccumulatorCoder()));
  }
  return DoFnInfo.forFn(
      doFn,
      combineFn.getWindowingStrategy(),
      combineFn.getSideInputViews(),
      inputCoder,
      Collections.emptyMap(), // Not needed here.
      new TupleTag<>(PropertyNames.OUTPUT),
      DoFnSchemaInformation.create(),
      Collections.emptyMap());
}
 

private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<
            PCollection<KV<K, Iterable<InputT>>>, ?, Combine.GroupedValues<K, InputT, ?>>
        transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<KV<K, Iterable<InputT>>> mainInput =
        (PCollection<KV<K, Iterable<InputT>>>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    KvCoder<K, Iterable<InputT>> kvCoder = (KvCoder<K, Iterable<InputT>>) mainInput.getCoder();
    IterableCoder<InputT> iterCoder = (IterableCoder<InputT>) kvCoder.getValueCoder();
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(), iterCoder.getElemCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 

@Test
public void registerCoder() throws IOException {
  Coder<?> coder =
      KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of())));
  String id = components.registerCoder(coder);
  assertThat(components.registerCoder(coder), equalTo(id));
  assertThat(id, not(isEmptyOrNullString()));
  Coder<?> equalCoder =
      KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of())));
  assertThat(components.registerCoder(equalCoder), equalTo(id));
  Coder<?> otherCoder = VarLongCoder.of();
  assertThat(components.registerCoder(otherCoder), not(equalTo(id)));

  components.toComponents().getCodersOrThrow(id);
  components.toComponents().getCodersOrThrow(components.registerCoder(otherCoder));
}
 

private KvCoder<K, InputT> getKvCoder(
    Coder<? extends KV<K, ? extends Iterable<InputT>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("Combine.GroupedValues requires its input to use KvCoder");
  }
  @SuppressWarnings({"unchecked", "rawtypes"})
  KvCoder<K, ? extends Iterable<InputT>> kvCoder = (KvCoder) inputCoder;
  Coder<K> keyCoder = kvCoder.getKeyCoder();
  Coder<? extends Iterable<InputT>> kvValueCoder = kvCoder.getValueCoder();
  if (!(kvValueCoder instanceof IterableCoder)) {
    throw new IllegalStateException(
        "Combine.GroupedValues requires its input values to use " + "IterableCoder");
  }
  @SuppressWarnings("unchecked")
  IterableCoder<InputT> inputValuesCoder = (IterableCoder<InputT>) kvValueCoder;
  Coder<InputT> inputValueCoder = inputValuesCoder.getElemCoder();
  return KvCoder.of(keyCoder, inputValueCoder);
}
 

@Test
public void windowedValueCoderComponentsToConstructor() throws IOException {
  FullWindowedValueCoder<Iterable<KV<String, Integer>>> javaCoder =
      FullWindowedValueCoder.of(
          IterableCoder.of(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())),
          IntervalWindowCoder.of());

  MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder);
  WindowedValueCoderComponents windowedValueCoderComponents =
      ModelCoders.getWindowedValueCoderComponents(coderAndComponents.getCoder());

  Coder windowedCoder =
      ModelCoders.windowedValueCoder(
          windowedValueCoderComponents.elementCoderId(),
          windowedValueCoderComponents.windowCoderId());

  assertThat(windowedCoder, equalTo(coderAndComponents.getCoder()));
}
 

@Override
public PCollection<KV<K, Iterable<V>>> expand(
    PCollection<KV<K, Iterable<WindowedValue<V>>>> input) {
  @SuppressWarnings("unchecked")
  KvCoder<K, Iterable<WindowedValue<V>>> inputKvCoder =
      (KvCoder<K, Iterable<WindowedValue<V>>>) input.getCoder();

  Coder<K> keyCoder = inputKvCoder.getKeyCoder();
  Coder<Iterable<WindowedValue<V>>> inputValueCoder = inputKvCoder.getValueCoder();

  IterableCoder<WindowedValue<V>> inputIterableValueCoder =
      (IterableCoder<WindowedValue<V>>) inputValueCoder;
  Coder<WindowedValue<V>> inputIterableElementCoder = inputIterableValueCoder.getElemCoder();
  WindowedValueCoder<V> inputIterableWindowedValueCoder =
      (WindowedValueCoder<V>) inputIterableElementCoder;

  Coder<V> inputIterableElementValueCoder = inputIterableWindowedValueCoder.getValueCoder();
  Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder);
  Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder);

  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder);
}
 

/** Retrieves the {@link Coder} for the secondary key-value pairs. */
@SuppressWarnings("unchecked")
private static <PrimaryKeyT, SecondaryKeyT, ValueT>
    KvCoder<SecondaryKeyT, ValueT> getSecondaryKeyValueCoder(
        Coder<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> inputCoder) {
  if (!(inputCoder instanceof KvCoder)) {
    throw new IllegalStateException("SortValues requires its input to use KvCoder");
  }
  @SuppressWarnings("unchecked")
  KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>> kvCoder =
      (KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>) inputCoder;

  if (!(kvCoder.getValueCoder() instanceof IterableCoder)) {
    throw new IllegalStateException(
        "SortValues requires the values be encoded with IterableCoder");
  }
  IterableCoder<KV<SecondaryKeyT, ValueT>> iterableCoder =
      (IterableCoder<KV<SecondaryKeyT, ValueT>>) (kvCoder.getValueCoder());

  if (!(iterableCoder.getElemCoder() instanceof KvCoder)) {
    throw new IllegalStateException(
        "SortValues requires the secondary key-value pairs to use KvCoder");
  }
  return (KvCoder<SecondaryKeyT, ValueT>) (iterableCoder.getElemCoder());
}
 

@Test
public void testBufferingGroupingTable() throws Exception {
  GroupingTableBase<String, String, List<String>> table =
      (GroupingTableBase<String, String, List<String>>)
          GroupingTables.buffering(
              new IdentityGroupingKeyCreator(), new KvPairInfo(),
              new StringPowerSizeEstimator(), new StringPowerSizeEstimator());
  table.setMaxSize(1000);
  TestOutputReceiver receiver =
      new TestOutputReceiver(
          KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(StringUtf8Coder.of())),
          NameContextsForTests.nameContextForTest());

  table.put("A", "a", receiver);
  table.put("B", "b1", receiver);
  table.put("B", "b2", receiver);
  table.put("C", "c", receiver);
  assertThat(receiver.outputElems, empty());

  table.put("C", "cccc", receiver);
  assertThat(receiver.outputElems, hasItem((Object) KV.of("C", Arrays.asList("c", "cccc"))));

  table.put("DDDD", "d", receiver);
  assertThat(receiver.outputElems, hasItem((Object) KV.of("DDDD", Arrays.asList("d"))));

  table.flush(receiver);
  assertThat(
      receiver.outputElems,
      IsIterableContainingInAnyOrder.<Object>containsInAnyOrder(
          KV.of("A", Arrays.asList("a")),
          KV.of("B", Arrays.asList("b1", "b2")),
          KV.of("C", Arrays.asList("c", "cccc")),
          KV.of("DDDD", Arrays.asList("d"))));
}
 

static ParallelInstruction createPartialGroupByKeyInstruction(
    int producerIndex, int producerOutputNum) {
  InstructionInput cloudInput = new InstructionInput();
  cloudInput.setProducerInstructionIndex(producerIndex);
  cloudInput.setOutputNum(producerOutputNum);

  PartialGroupByKeyInstruction pgbkInstruction = new PartialGroupByKeyInstruction();
  pgbkInstruction.setInput(cloudInput);
  pgbkInstruction.setInputElementCodec(
      CloudObjects.asCloudObject(
          FullWindowedValueCoder.of(
              KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()),
              IntervalWindowCoder.of()),
          /*sdkComponents=*/ null));

  InstructionOutput output = new InstructionOutput();
  output.setName("pgbk_output_name");
  output.setCodec(
      CloudObjects.asCloudObject(
          KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(BigEndianIntegerCoder.of())),
          /*sdkComponents=*/ null));
  output.setOriginalName("originalName");
  output.setSystemName("systemName");

  ParallelInstruction instruction = new ParallelInstruction();
  instruction.setOriginalName("pgbk_original_name");
  instruction.setSystemName("pgbk_system_name");
  instruction.setPartialGroupByKey(pgbkInstruction);
  instruction.setOutputs(Arrays.asList(output));

  return instruction;
}
 

@Test
public void testConstruction() {
  String transformId = "my_target_pt";

  KvCoder<byte[], Iterable<Long>> coder =
      KvCoder.of(LengthPrefixCoder.of(ByteArrayCoder.of()), IterableCoder.of(VarLongCoder.of()));
  RemoteInputDestination<KV<byte[], Iterable<Long>>> destination =
      RemoteInputDestination.of(coder, transformId);

  assertThat(destination.getCoder(), equalTo(coder));
  assertThat(destination.getPTransformId(), equalTo(transformId));
}
 

@Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) {
  KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder());
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      outputWindowingStrategy,
      input.isBounded(),
      KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder())));
}
 

public Coder<V> getValueCoder(Coder<KV<K, Iterable<WindowedValue<V>>>> inputCoder) {
  // Coder<Iterable<...>> --> IterableCoder<...>
  Coder<Iterable<WindowedValue<V>>> iterableWindowedValueCoder =
      getKvCoder(inputCoder).getValueCoder();
  checkArgument(
      iterableWindowedValueCoder instanceof IterableCoder,
      "%s requires a %s<..., %s> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      iterableWindowedValueCoder);
  IterableCoder<WindowedValue<V>> iterableCoder =
      (IterableCoder<WindowedValue<V>>) iterableWindowedValueCoder;

  // Coder<WindowedValue<...>> --> WindowedValueCoder<...>
  Coder<WindowedValue<V>> iterableElementCoder = iterableCoder.getElemCoder();
  checkArgument(
      iterableElementCoder instanceof WindowedValueCoder,
      "%s requires a %s<..., %s<%s>> but got a %s",
      getClass().getSimpleName(),
      KvCoder.class.getSimpleName(),
      IterableCoder.class.getSimpleName(),
      WindowedValueCoder.class.getSimpleName(),
      iterableElementCoder);
  WindowedValueCoder<V> windowedValueCoder = (WindowedValueCoder<V>) iterableElementCoder;

  return windowedValueCoder.getValueCoder();
}
 

private static <ReadT, WriteT>
    TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>> createPCollView() {
  return new TransformEvaluator<View.CreatePCollectionView<ReadT, WriteT>>() {
    @Override
    public void evaluate(
        View.CreatePCollectionView<ReadT, WriteT> transform, EvaluationContext context) {
      Iterable<? extends WindowedValue<?>> iter =
          context.getWindowedValues(context.getInput(transform));
      PCollectionView<WriteT> output = transform.getView();
      Coder<Iterable<WindowedValue<?>>> coderInternal =
          (Coder)
              IterableCoder.of(
                  WindowedValue.getFullCoder(
                      output.getCoderInternal(),
                      output.getWindowingStrategyInternal().getWindowFn().windowCoder()));

      @SuppressWarnings("unchecked")
      Iterable<WindowedValue<?>> iterCast = (Iterable<WindowedValue<?>>) iter;

      context.putPView(output, iterCast, coderInternal);
    }

    @Override
    public String toNativeString() {
      return "<createPCollectionView>";
    }
  };
}
 

private static <K, InputT>
    FullWindowedValueCoder<KV<K, Iterable<InputT>>> windowedValueKeyValueCoderOf(
        final Coder<K> keyCoder,
        final Coder<InputT> iCoder,
        final Coder<? extends BoundedWindow> wCoder) {
  return FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder);
}
 

@Override
public void translateTransform(
    PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Iterable<V>>>> transform,
    TranslationContext context) {

  @SuppressWarnings("unchecked")
  final PCollection<KV<K, V>> inputPCollection = (PCollection<KV<K, V>>) context.getInput();
  Dataset<WindowedValue<KV<K, V>>> input = context.getDataset(inputPCollection);
  WindowingStrategy<?, ?> windowingStrategy = inputPCollection.getWindowingStrategy();
  KvCoder<K, V> kvCoder = (KvCoder<K, V>) inputPCollection.getCoder();
  Coder<V> valueCoder = kvCoder.getValueCoder();

  // group by key only
  Coder<K> keyCoder = kvCoder.getKeyCoder();
  KeyValueGroupedDataset<K, WindowedValue<KV<K, V>>> groupByKeyOnly =
      input.groupByKey(KVHelpers.extractKey(), EncoderHelpers.fromBeamCoder(keyCoder));

  // group also by windows
  WindowedValue.FullWindowedValueCoder<KV<K, Iterable<V>>> outputCoder =
      WindowedValue.FullWindowedValueCoder.of(
          KvCoder.of(keyCoder, IterableCoder.of(valueCoder)),
          windowingStrategy.getWindowFn().windowCoder());
  Dataset<WindowedValue<KV<K, Iterable<V>>>> output =
      groupByKeyOnly.flatMapGroups(
          new GroupAlsoByWindowViaOutputBufferFn<>(
              windowingStrategy,
              new InMemoryStateInternalsFactory<>(),
              SystemReduceFn.buffering(valueCoder),
              context.getSerializableOptions()),
          EncoderHelpers.fromBeamCoder(outputCoder));

  context.putDataset(context.getOutput(), output);
}
 

/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link
 * TriggerStateMachine}, for mocking the interactions between {@link ReduceFnRunner} and the
 * {@link TriggerStateMachine}.
 *
 * <p>Ignores the {@link Trigger} on the {@link WindowingStrategy}.
 */
public static <W extends BoundedWindow>
    ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining(
        WindowingStrategy<?, W> windowingStrategy, TriggerStateMachine triggerStateMachine)
        throws Exception {
  return new ReduceFnTester<>(
      windowingStrategy,
      triggerStateMachine,
      SystemReduceFn.buffering(VarIntCoder.of()),
      IterableCoder.of(VarIntCoder.of()),
      PipelineOptionsFactory.create(),
      NullSideInputReader.empty());
}
 

/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, creating a {@link
 * TriggerStateMachine} from its {@link Trigger}.
 */
public static <W extends BoundedWindow>
    ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining(
        WindowingStrategy<?, W> windowingStrategy) throws Exception {
  return new ReduceFnTester<>(
      windowingStrategy,
      TriggerStateMachines.stateMachineForTrigger(
          TriggerTranslation.toProto(windowingStrategy.getTrigger())),
      SystemReduceFn.buffering(VarIntCoder.of()),
      IterableCoder.of(VarIntCoder.of()),
      PipelineOptionsFactory.create(),
      NullSideInputReader.empty());
}
 

private KeyedWorkItemCoder(
    Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) {
  this.keyCoder = keyCoder;
  this.elemCoder = elemCoder;
  this.windowCoder = windowCoder;
  this.timersCoder = IterableCoder.of(TimerDataCoderV2.of(windowCoder));
  this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder));
}