org.apache.beam.sdk.values.TimestampedValue Java Examples

@Override
public PCollection<TimestampedValue<T>> expand(PCollection<T> input) {
  return input
      .apply(
          ParDo.of(
              new DoFn<T, TimestampedValue<T>>() {
                @ProcessElement
                public void processElement(
                    @Element T element,
                    @Timestamp Instant timestamp,
                    OutputReceiver<TimestampedValue<T>> r) {
                  r.output(TimestampedValue.of(element, timestamp));
                }
              }))
      .setCoder(TimestampedValueCoder.of(input.getCoder()));
}
 

@Override
public PCollection<Row> buildIOReader(PBegin begin) {
  TestStream.Builder<Row> values = TestStream.create(schema);

  for (Pair<Duration, List<Row>> pair : timestampedRows) {
    values = values.advanceWatermarkTo(new Instant(0).plus(pair.getKey()));
    for (int i = 0; i < pair.getValue().size(); i++) {
      values =
          values.addElements(
              TimestampedValue.of(
                  pair.getValue().get(i),
                  new Instant(pair.getValue().get(i).getDateTime(timestampField))));
    }
  }

  return begin
      .apply(
          "MockedUnboundedTable_" + COUNTER.incrementAndGet(),
          values.advanceWatermarkToInfinity())
      .setRowSchema(getSchema());
}
 

@Test
@Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class})
public void testFixedWindowsCombine() {
  PCollection<KV<String, Integer>> input =
      pipeline
          .apply(
              Create.timestamped(
                      TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
                      TimestampedValue.of(KV.of("a", 1), new Instant(1L)),
                      TimestampedValue.of(KV.of("a", 4), new Instant(6L)),
                      TimestampedValue.of(KV.of("b", 1), new Instant(7L)),
                      TimestampedValue.of(KV.of("b", 13), new Instant(8L)))
                  .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
          .apply(Window.into(FixedWindows.of(Duration.millis(2))));

  PCollection<Integer> sum =
      input.apply(Values.create()).apply(Combine.globally(new SumInts()).withoutDefaults());

  PCollection<KV<String, String>> sumPerKey = input.apply(Combine.perKey(new TestCombineFn()));

  PAssert.that(sum).containsInAnyOrder(2, 5, 13);
  PAssert.that(sumPerKey)
      .containsInAnyOrder(
          Arrays.asList(KV.of("a", "11"), KV.of("a", "4"), KV.of("b", "1"), KV.of("b", "13")));
  pipeline.run();
}
 

public static void main(String[] args) {
  PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create();
  Pipeline pipeline = Pipeline.create(options);

  PCollection<String> events =
      pipeline.apply(
          Create.timestamped(
              TimestampedValue.of("event", Instant.parse("2019-06-01T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-01T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-01T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-01T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-05T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-05T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-08T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-08T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-08T00:00:00+00:00")),
              TimestampedValue.of("event", Instant.parse("2019-06-10T00:00:00+00:00"))
          )
      );

  PCollection<KV<String, Long>> output = applyTransform(events);

  output.apply(Log.ofElements());

  pipeline.run();
}
 

/**
 * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with
 * the windowing function customized to actually be the same as the default, the earlier of the
 * two values.
 */
@Test
@Category(ValidatesRunner.class)
public void testTimestampCombinerEarliest() {

  p.apply(
          Create.timestamped(
              TimestampedValue.of(KV.of(0, "hello"), new Instant(0)),
              TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10))))
      .apply(
          Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10)))
              .withTimestampCombiner(TimestampCombiner.EARLIEST))
      .apply(GroupByKey.create())
      .apply(ParDo.of(new AssertTimestamp(new Instant(0))));

  p.run();
}
 

@Test
@Category({
  ValidatesRunner.class,
  UsesTimersInParDo.class,
  UsesStatefulParDo.class,
  UsesStrictTimerOrdering.class
})
public void testEventTimeTimerOrderingWithCreate() throws Exception {
  final int numTestElements = 100;
  final Instant now = new Instant(1500000000000L);

  List<TimestampedValue<KV<String, String>>> elements = new ArrayList<>();
  for (int i = 0; i < numTestElements; i++) {
    elements.add(TimestampedValue.of(KV.of("dummy", "" + i), now.plus(i * 1000)));
  }

  testEventTimeTimerOrderingWithInputPTransform(
      now, numTestElements, Create.timestamped(elements));
}
 

@Test
@Category(ValidatesRunner.class)
public void testNonPartitioningWindowing() {
  PCollection<String> input =
      p.apply(
          Create.timestamped(
              TimestampedValue.of("a", new Instant(1)),
              TimestampedValue.of("a", new Instant(7)),
              TimestampedValue.of("b", new Instant(8))));

  PCollection<String> output =
      input.apply(new WindowedCount(SlidingWindows.of(new Duration(10)).every(new Duration(5))));

  PAssert.that(output)
      .containsInAnyOrder(
          output("a", 1, 1, -5, 5),
          output("a", 2, 5, 0, 10),
          output("a", 1, 10, 5, 15),
          output("b", 1, 8, 0, 10),
          output("b", 1, 10, 5, 15));

  p.run();
}
 

@Test
public void testBinaryCombineWithSlidingWindows() {
  PCollection<Integer> input =
      pipeline
          .apply(
              Create.timestamped(
                  TimestampedValue.of(1, new Instant(1)),
                  TimestampedValue.of(3, new Instant(2)),
                  TimestampedValue.of(5, new Instant(3))))
          .apply(Window.into(SlidingWindows.of(Duration.millis(3)).every(Duration.millis(1))))
          .apply(
              Combine.globally(
                      Combine.BinaryCombineFn.of(
                          (SerializableBiFunction<Integer, Integer, Integer>)
                              (integer1, integer2) -> integer1 > integer2 ? integer1 : integer2))
                  .withoutDefaults());
  PAssert.that(input).containsInAnyOrder(1, 3, 5, 5, 5);
  pipeline.run();
}
 

@Test
public void testCombinePerKeyPreservesWindowing() {
  PCollection<KV<Integer, Integer>> input =
      pipeline
          .apply(
              Create.timestamped(
                  TimestampedValue.of(KV.of(1, 1), new Instant(1)),
                  TimestampedValue.of(KV.of(1, 3), new Instant(2)),
                  TimestampedValue.of(KV.of(1, 5), new Instant(11)),
                  TimestampedValue.of(KV.of(2, 2), new Instant(3)),
                  TimestampedValue.of(KV.of(2, 4), new Instant(11)),
                  TimestampedValue.of(KV.of(2, 6), new Instant(12))))
          .apply(Window.into(FixedWindows.of(Duration.millis(10))))
          .apply(Sum.integersPerKey());
  PAssert.that(input).containsInAnyOrder(KV.of(1, 4), KV.of(1, 5), KV.of(2, 2), KV.of(2, 10));
  pipeline.run();
}
 

@Test
@Category({ValidatesRunner.class, UsesTestStream.class})
public void testReshuffleWithTimestampsStreaming() {
  TestStream<Long> stream =
      TestStream.create(VarLongCoder.of())
          .advanceWatermarkTo(new Instant(0L).plus(Duration.standardDays(48L)))
          .addElements(
              TimestampedValue.of(0L, new Instant(0L)),
              TimestampedValue.of(1L, new Instant(0L).plus(Duration.standardDays(48L))),
              TimestampedValue.of(
                  2L, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(48L))))
          .advanceWatermarkToInfinity();
  PCollection<KV<String, Long>> input =
      pipeline
          .apply(stream)
          .apply(WithKeys.of(""))
          .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10L))));

  PCollection<KV<String, Long>> reshuffled = input.apply(Reshuffle.of());
  PAssert.that(reshuffled.apply(Values.create())).containsInAnyOrder(0L, 1L, 2L);

  pipeline.run();
}
 

@Test
public void testCombinePerKeyWithSlidingWindows() {
  PCollection<KV<Integer, Integer>> input =
      pipeline
          .apply(
              Create.timestamped(
                  TimestampedValue.of(KV.of(1, 1), new Instant(1)),
                  TimestampedValue.of(KV.of(1, 3), new Instant(2)),
                  TimestampedValue.of(KV.of(1, 5), new Instant(3)),
                  TimestampedValue.of(KV.of(1, 2), new Instant(1)),
                  TimestampedValue.of(KV.of(1, 4), new Instant(2)),
                  TimestampedValue.of(KV.of(1, 6), new Instant(3))))
          .apply(Window.into(SlidingWindows.of(Duration.millis(3)).every(Duration.millis(1))))
          .apply(Sum.integersPerKey());
  PAssert.that(input)
      .containsInAnyOrder(
          KV.of(1, 1 + 2),
          KV.of(1, 1 + 2 + 3 + 4),
          KV.of(1, 1 + 3 + 5 + 2 + 4 + 6),
          KV.of(1, 3 + 4 + 5 + 6),
          KV.of(1, 5 + 6));
  pipeline.run();
}
 

@Override
public TimestampedValue<T> mergeAccumulators(Iterable<TimestampedValue<T>> accumulators) {
  checkNotNull(accumulators, "accumulators must be non-null");

  Iterator<TimestampedValue<T>> iter = accumulators.iterator();
  if (!iter.hasNext()) {
    return createAccumulator();
  }

  TimestampedValue<T> merged = iter.next();
  while (iter.hasNext()) {
    merged = addInput(merged, iter.next());
  }

  return merged;
}
 

@Override
protected Iterable<TimestampedValue<CategoryPrice>> relevantResults(
    Iterable<TimestampedValue<CategoryPrice>> results) {
  // Find the last (in processing time) reported average price for each category.
  Map<Long, TimestampedValue<CategoryPrice>> finalAverages = new TreeMap<>();
  for (TimestampedValue<CategoryPrice> obj : results) {
    Assert.assertTrue("have CategoryPrice", obj.getValue() instanceof CategoryPrice);
    CategoryPrice categoryPrice = (CategoryPrice) obj.getValue();
    if (categoryPrice.isLast) {
      finalAverages.put(
          categoryPrice.category, TimestampedValue.of(categoryPrice, obj.getTimestamp()));
    }
  }

  return finalAverages.values();
}
 

@Test
public void testNonPollingGrowthTrackerCheckpointNonEmpty() {
  Instant now = Instant.now();
  PollResult<String> claim =
      PollResult.incomplete(
              Arrays.asList(
                  TimestampedValue.of("d", now.plus(standardSeconds(4))),
                  TimestampedValue.of("c", now.plus(standardSeconds(3))),
                  TimestampedValue.of("a", now.plus(standardSeconds(1))),
                  TimestampedValue.of("b", now.plus(standardSeconds(2)))))
          .withWatermark(now.plus(standardSeconds(7)));

  GrowthTracker<String, Integer> tracker = newTracker(NonPollingGrowthState.of(claim));

  assertTrue(tracker.tryClaim(KV.of(claim, 1 /* termination state */)));
  GrowthState residual = tracker.trySplit(0).getResidual();
  NonPollingGrowthState<String> primary =
      (NonPollingGrowthState<String>) tracker.currentRestriction();
  tracker.checkDone();

  // Verify primary: should contain what the current tracker claimed, and nothing else.
  assertEquals(claim, primary.getPending());

  // Verify residual: should contain what the current tracker didn't claim.
  assertEquals(GrowthTracker.EMPTY_STATE, residual);
}
 

@Override
public PollResult<OutputT> apply(InputT element, Context c) throws Exception {
  Instant now = Instant.now();
  Duration elapsed = new Duration(baseTime, Instant.now());
  if (elapsed.isLongerThan(timeToFail)) {
    fail(
        String.format(
            "Poll called %s after base time, which is longer than the threshold of %s",
            elapsed, timeToFail));
  }

  double fractionElapsed = 1.0 * elapsed.getMillis() / timeToOutputEverything.getMillis();
  int numToEmit = (int) Math.min(outputs.size(), fractionElapsed * outputs.size());
  List<TimestampedValue<OutputT>> toEmit = Lists.newArrayList();
  for (int i = 0; i < numToEmit; ++i) {
    toEmit.add(TimestampedValue.of(outputs.get(i), now));
  }
  return elapsed.isLongerThan(timeToDeclareOutputFinal)
      ? PollResult.complete(toEmit)
      : PollResult.incomplete(toEmit).withWatermark(now);
}
 

/** Test {@code query} matches {@code model}. */
private <T extends KnownSize> void queryMatchesModel(
    String name,
    NexmarkQueryTransform<T> query,
    NexmarkQueryModel<T> model,
    boolean streamingMode) {
  NexmarkUtils.setupPipeline(NexmarkUtils.CoderStrategy.HAND, p);

  PCollection<Event> events =
      p.apply(
          name + ".Read",
          streamingMode
              ? NexmarkUtils.streamEventsSource(CONFIG)
              : NexmarkUtils.batchEventsSource(CONFIG));
  PCollection<TimestampedValue<T>> results =
      (PCollection<TimestampedValue<T>>) events.apply(new NexmarkQuery<>(CONFIG, query));
  PAssert.that(results).satisfies(model.assertionFor());
  PipelineResult result = p.run();
  result.waitUntilFinish();
}
 

/**
 * A smoke test that the count of input bids and outputs are the same, to help diagnose
 * flakiness in more complex tests.
 */
@Test
public void inputOutputSameEvents() throws Exception {
  NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
  config.sideInputType = NexmarkUtils.SideInputType.DIRECT;
  config.numEventGenerators = 1;
  config.numEvents = 5000;
  config.sideInputRowCount = 10;
  config.sideInputNumShards = 3;
  PCollection<KV<Long, String>> sideInput = NexmarkUtils.prepareSideInput(p, config);

  try {
    PCollection<Event> input = p.apply(NexmarkUtils.batchEventsSource(config));
    PCollection<Bid> justBids = input.apply(NexmarkQueryUtil.JUST_BIDS);
    PCollection<Long> bidCount = justBids.apply("Count Bids", Count.globally());

    NexmarkQueryTransform<Bid> query = getQuery(config);
    query.setSideInput(sideInput);

    PCollection<TimestampedValue<Bid>> output =
        (PCollection<TimestampedValue<Bid>>) input.apply(new NexmarkQuery(config, query));
    PCollection<Long> outputCount = output.apply("Count outputs", Count.globally());

    PAssert.that(PCollectionList.of(bidCount).and(outputCount).apply(Flatten.pCollections()))
        .satisfies(
            counts -> {
              assertThat(Iterables.size(counts), equalTo(2));
              assertThat(Iterables.get(counts, 0), greaterThan(0L));
              assertThat(Iterables.get(counts, 0), equalTo(Iterables.get(counts, 1)));
              return null;
            });
    p.run();
  } finally {
    NexmarkUtils.cleanUpSideInput(config);
  }
}
 

@Test
public void testElementAtPositiveInfinityThrows() {
  Builder<Integer> stream =
      TestStream.create(VarIntCoder.of())
          .addElements(TimestampedValue.of(-1, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(1L)));
  thrown.expect(IllegalArgumentException.class);
  stream.addElements(TimestampedValue.of(1, BoundedWindow.TIMESTAMP_MAX_VALUE));
}
 

/** Test {@code query} matches {@code model}. */
private <T extends KnownSize> void queryMatchesModel(
    String name,
    NexmarkConfiguration config,
    NexmarkQueryTransform<T> query,
    NexmarkQueryModel<T> model,
    boolean streamingMode)
    throws Exception {

  ResourceId sideInputResourceId =
      FileSystems.matchNewResource(
          String.format(
              "%s/SessionSideInputJoin-%s",
              p.getOptions().getTempLocation(), new Random().nextInt()),
          false);
  config.sideInputUrl = sideInputResourceId.toString();

  try {
    PCollection<KV<Long, String>> sideInput = NexmarkUtils.prepareSideInput(p, config);
    query.setSideInput(sideInput);

    PCollection<Event> events =
        p.apply(
            name + ".Read",
            streamingMode
                ? NexmarkUtils.streamEventsSource(config)
                : NexmarkUtils.batchEventsSource(config));

    PCollection<TimestampedValue<T>> results =
        (PCollection<TimestampedValue<T>>) events.apply(new NexmarkQuery<>(config, query));
    PAssert.that(results).satisfies(model.assertionFor());
    PipelineResult result = p.run();
    result.waitUntilFinish();
  } finally {
    NexmarkUtils.cleanUpSideInput(config);
  }
}
 

@Override
protected void run() {
  TimestampedValue<Event> timestampedEvent = nextInput();
  if (timestampedEvent == null) {
    for (Long bidder : ImmutableSet.copyOf(activeSessions.keySet())) {
      flushSession(bidder);
    }
    allDone();
    return;
  }
  Event event = timestampedEvent.getValue();
  if (event.bid == null) {
    // Ignore non-bid events.
    return;
  }

  List<TimestampedValue<Event>> activeSession = activeSessions.get(event.bid.bidder);
  if (activeSession == null) {
    beginSession(timestampedEvent);
  } else if (timestampedEvent
      .getTimestamp()
      .isAfter(
          activeSession
              .get(activeSession.size() - 1)
              .getTimestamp()
              .plus(configuration.sessionGap))) {
    flushSession(event.bid.bidder);
    beginSession(timestampedEvent);
  } else {
    activeSession.add(timestampedEvent);
  }
}
 

/** Ensure a closed trigger has its state recorded in the merge result window. */
@Test
public void testMergingWithCloseTrigger() throws Exception {
  Duration allowedLateness = Duration.millis(50);
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          Sessions.withGapDuration(Duration.millis(10)),
          mockTriggerStateMachine,
          AccumulationMode.DISCARDING_FIRED_PANES,
          allowedLateness,
          ClosingBehavior.FIRE_IF_NON_EMPTY);

  // Create a new merged session window.
  IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12));
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));

  // Force the trigger to be closed for the merged window.
  when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
  triggerShouldFinish(mockTriggerStateMachine);

  // Fire and end-of-window timer as though the trigger set it
  tester.advanceInputWatermark(new Instant(13));
  tester.fireTimer(mergedWindow, mergedWindow.maxTimestamp(), TimeDomain.EVENT_TIME);

  // Trigger is now closed.
  assertTrue(tester.isMarkedFinished(mergedWindow));

  when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);

  // Revisit the same session window.
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));

  // Trigger is still closed.
  assertTrue(tester.isMarkedFinished(mergedWindow));
}
 

@SafeVarargs
private final <T> CommittedBundle<T> timestampedBundle(
    PCollection<T> pc, TimestampedValue<T>... values) {
  UncommittedBundle<T> bundle = bundleFactory.createBundle(pc);
  for (TimestampedValue<T> value : values) {
    bundle.add(
        WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp()));
  }
  return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
}
 

/** Test {@code query} matches {@code model}. */
private <T extends KnownSize> void queryMatchesModel(
    String name,
    NexmarkConfiguration config,
    NexmarkQueryTransform<T> query,
    NexmarkQueryModel<T> model,
    boolean streamingMode)
    throws Exception {

  ResourceId sideInputResourceId =
      FileSystems.matchNewResource(
          String.format(
              "%s/BoundedSideInputJoin-%s",
              p.getOptions().getTempLocation(), new Random().nextInt()),
          false);
  config.sideInputUrl = sideInputResourceId.toString();

  try {
    PCollection<KV<Long, String>> sideInput = NexmarkUtils.prepareSideInput(p, config);
    query.setSideInput(sideInput);

    PCollection<Event> events =
        p.apply(
            name + ".Read",
            streamingMode
                ? NexmarkUtils.streamEventsSource(config)
                : NexmarkUtils.batchEventsSource(config));

    PCollection<TimestampedValue<T>> results =
        (PCollection<TimestampedValue<T>>) events.apply(new NexmarkQuery<>(config, query));
    PAssert.that(results).satisfies(model.assertionFor());
    PipelineResult result = p.run();
    result.waitUntilFinish();
  } finally {
    NexmarkUtils.cleanUpSideInput(config);
  }
}
 

@Test
@Category(NeedsRunner.class)
public void testGloballyEventTimestamp() {
  PCollection<String> output =
      p.apply(
              Create.timestamped(
                  TimestampedValue.of("foo", new Instant(100)),
                  TimestampedValue.of("bar", new Instant(300)),
                  TimestampedValue.of("baz", new Instant(200))))
          .apply(Latest.globally());

  PAssert.that(output).containsInAnyOrder("bar");
  p.run();
}
 

Reader(
    @Nullable List<TimestampedValue<T>> residualElementsList,
    @Nullable BoundedSource<T> residualSource,
    PipelineOptions options) {
  init(residualElementsList, residualSource, options);
  this.options = checkNotNull(options, "options");
  this.done = false;
}
 

@Test
public void testAddInputSameTimestamp() {
  TimestampedValue<Long> accum = TimestampedValue.of(100L, INSTANT);
  TimestampedValue<Long> input = TimestampedValue.of(200L, INSTANT);

  assertThat(
      "Latest for values with the same timestamp is chosen arbitrarily",
      fn.addInput(accum, input),
      isOneOf(accum, input));
}
 

/**
 * Tests that when a processing time timers comes in after a window is expired it does not cause
 * a spurious output.
 */
@Test
@Category({ValidatesRunner.class, UsesTestStreamWithProcessingTime.class})
public void testCombiningAccumulatingProcessingTime() throws Exception {
  PCollection<Integer> triggeredSums =
      p.apply(
              TestStream.create(VarIntCoder.of())
                  .advanceWatermarkTo(new Instant(0))
                  .addElements(
                      TimestampedValue.of(2, new Instant(2)),
                      TimestampedValue.of(5, new Instant(5)))
                  .advanceWatermarkTo(new Instant(100))
                  .advanceProcessingTime(Duration.millis(10))
                  .advanceWatermarkToInfinity())
          .apply(
              Window.<Integer>into(FixedWindows.of(Duration.millis(100)))
                  .withTimestampCombiner(TimestampCombiner.EARLIEST)
                  .accumulatingFiredPanes()
                  .withAllowedLateness(Duration.ZERO)
                  .triggering(
                      Repeatedly.forever(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(Duration.millis(10)))))
          .apply(Sum.integersGlobally().withoutDefaults());

  PAssert.that(triggeredSums).containsInAnyOrder(7);

  p.run();
}
 

@Override
public void processElement(WindowedValue<TestStreamIndex<T>> element) throws Exception {
  TestStreamIndex<T> streamIndex = element.getValue();
  List<Event<T>> events = streamIndex.getTestStream().getEvents();
  int index = streamIndex.getIndex();
  Instant watermark = element.getTimestamp();
  Event<T> event = events.get(index);

  if (event.getType().equals(EventType.ELEMENT)) {
    UncommittedBundle<T> bundle =
        context.createBundle(
            (PCollection<T>) Iterables.getOnlyElement(application.getOutputs().values()));
    for (TimestampedValue<T> elem : ((ElementEvent<T>) event).getElements()) {
      bundle.add(
          WindowedValue.timestampedValueInGlobalWindow(elem.getValue(), elem.getTimestamp()));
    }
    resultBuilder.addOutput(bundle);
  }

  if (event.getType().equals(EventType.WATERMARK)) {
    watermark = ((WatermarkEvent<T>) event).getWatermark();
  }

  if (event.getType().equals(EventType.PROCESSING_TIME)) {
    ((TestClock) context.getClock())
        .advance(((ProcessingTimeEvent<T>) event).getProcessingTimeAdvance());
  }

  TestStreamIndex<T> next = streamIndex.next();
  if (next.getIndex() < events.size()) {
    resultBuilder.addUnprocessedElements(
        Collections.singleton(WindowedValue.timestampedValueInGlobalWindow(next, watermark)));
  }
}
 

static <T> RunnerApi.TestStreamPayload.Event eventToProto(
    TestStream.Event<T> event, Coder<T> coder) throws IOException {
  switch (event.getType()) {
    case WATERMARK:
      return RunnerApi.TestStreamPayload.Event.newBuilder()
          .setWatermarkEvent(
              RunnerApi.TestStreamPayload.Event.AdvanceWatermark.newBuilder()
                  .setNewWatermark(
                      ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis()))
          .build();

    case PROCESSING_TIME:
      return RunnerApi.TestStreamPayload.Event.newBuilder()
          .setProcessingTimeEvent(
              RunnerApi.TestStreamPayload.Event.AdvanceProcessingTime.newBuilder()
                  .setAdvanceDuration(
                      ((TestStream.ProcessingTimeEvent<T>) event)
                          .getProcessingTimeAdvance()
                          .getMillis()))
          .build();

    case ELEMENT:
      RunnerApi.TestStreamPayload.Event.AddElements.Builder builder =
          RunnerApi.TestStreamPayload.Event.AddElements.newBuilder();
      for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) {
        builder.addElements(
            RunnerApi.TestStreamPayload.TimestampedElement.newBuilder()
                .setTimestamp(element.getTimestamp().getMillis())
                .setEncodedElement(
                    ByteString.copyFrom(
                        CoderUtils.encodeToByteArray(coder, element.getValue()))));
      }
      return RunnerApi.TestStreamPayload.Event.newBuilder().setElementEvent(builder).build();
    default:
      throw new IllegalArgumentException(
          String.format(
              "Unsupported type of %s: %s",
              TestStream.Event.class.getCanonicalName(), event.getType()));
  }
}
 

@Test
public void testRequiresTimeSortedInputWithLateData() {
  Instant now = Instant.ofEpochMilli(0);
  PCollection<KV<String, Integer>> input =
      pipeline.apply(
          TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))
              .addElements(TimestampedValue.of(KV.of("", 1), now.plus(2)))
              .addElements(TimestampedValue.of(KV.of("", 2), now.plus(1)))
              .advanceWatermarkTo(now.plus(1))
              .addElements(TimestampedValue.of(KV.of("", 3), now))
              .advanceWatermarkToInfinity());
  PCollection<String> result = input.apply(ParDo.of(statefulConcat()));
  PAssert.that(result).containsInAnyOrder("2", "2:1");
  pipeline.run();
}