Java Code Examples for org.apache.beam.sdk.util.WindowedValue#getTimestamp()

WindowedValue<T> assignWindows(WindowedValue<T> input) throws Exception {
  // TODO: BEAM-4272 consider allocating only once and updating the current value per call.
  WindowFn<T, W>.AssignContext ctxt =
      windowFn.new AssignContext() {
        @Override
        public T element() {
          return input.getValue();
        }

        @Override
        public Instant timestamp() {
          return input.getTimestamp();
        }

        @Override
        public BoundedWindow window() {
          return Iterables.getOnlyElement(input.getWindows());
        }
      };
  Collection<W> windows = windowFn.assignWindows(ctxt);
  return WindowedValue.of(input.getValue(), input.getTimestamp(), windows, input.getPane());
}
 

@Override
public void processElement(WindowedValue<InT> inputElement, OpEmitter<OutT> emitter) {
  attemptStartBundle();

  final Iterable<WindowedValue<InT>> rejectedValues =
      pushbackFnRunner.processElementInReadyWindows(inputElement);
  for (WindowedValue<InT> rejectedValue : rejectedValues) {
    if (rejectedValue.getTimestamp().compareTo(pushbackWatermarkHold) < 0) {
      pushbackWatermarkHold = rejectedValue.getTimestamp();
    }
    pushbackValues.add(rejectedValue);
  }

  currentBundleElementCount.incrementAndGet();
  attemptFinishBundle(emitter);
}
 

@Override
public UncommittedImmutableListBundle<T> add(WindowedValue<T> element) {
  checkState(
      !committed,
      "Can't add element %s to committed bundle in PCollection %s",
      element,
      pcollection);
  checkArgument(
      element.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE),
      "Can't add an element past the end of time (%s), got timestamp %s",
      BoundedWindow.TIMESTAMP_MAX_VALUE,
      element.getTimestamp());
  elements.add(element);
  if (element.getTimestamp().isBefore(minSoFar)) {
    minSoFar = element.getTimestamp();
  }
  return this;
}
 

private <T> CommittedBundle<T> afterCommitGetElementsShouldHaveAddedElements(
    Iterable<WindowedValue<T>> elems) {
  UncommittedBundle<T> bundle = bundleFactory.createRootBundle();
  Collection<Matcher<? super WindowedValue<T>>> expectations = new ArrayList<>();
  Instant minElementTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<T> elem : elems) {
    bundle.add(elem);
    expectations.add(equalTo(elem));
    if (elem.getTimestamp().isBefore(minElementTs)) {
      minElementTs = elem.getTimestamp();
    }
  }
  Matcher<Iterable<? extends WindowedValue<T>>> containsMatcher =
      containsInAnyOrder(expectations);
  Instant commitTime = Instant.now();
  CommittedBundle<T> committed = bundle.commit(commitTime);
  assertThat(committed.getElements(), containsMatcher);

  // Sanity check that the test is meaningful.
  assertThat(minElementTs, not(equalTo(commitTime)));
  assertThat(committed.getMinimumTimestamp(), equalTo(minElementTs));
  assertThat(committed.getSynchronizedProcessingOutputWatermark(), equalTo(commitTime));

  return committed;
}
 

@Override
public void onData(final WindowedValue<T> windowedValue) {
  final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows());
  final T element = windowedValue.getValue();
  final Instant timestamp = windowedValue.getTimestamp();

  try {
    final Collection<W> windows =
      ((WindowFn<T, W>) windowFn)
        .assignWindows(
          ((WindowFn<T, W>) windowFn).new AssignContext() {
            @Override
            public T element() {
              return element;
            }

            @Override
            public Instant timestamp() {
              return timestamp;
            }

            @Override
            public BoundedWindow window() {
              return boundedWindow;
            }
          });

    // Emit compressed windows for efficiency
    outputCollector.emit(WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING));
  } catch (final Exception e) {
    throw new RuntimeException(e);
  }
}
 

private void processElementOrdered(BoundedWindow window, WindowedValue<InputT> value) {

    StateInternals stateInternals = stepContext.stateInternals();
    TimerInternals timerInternals = stepContext.timerInternals();

    Instant inputWatermark =
        MoreObjects.firstNonNull(
            timerInternals.currentInputWatermarkTime(), BoundedWindow.TIMESTAMP_MIN_VALUE);

    if (!inputWatermark.isAfter(
        value.getTimestamp().plus(windowingStrategy.getAllowedLateness()))) {

      StateNamespace namespace = StateNamespaces.window(windowCoder, window);
      BagState<WindowedValue<InputT>> sortBuffer = stateInternals.state(namespace, sortBufferTag);
      ValueState<Instant> minStampState = stateInternals.state(namespace, sortBufferMinStampTag);
      sortBuffer.add(value);
      Instant minStamp =
          MoreObjects.firstNonNull(minStampState.read(), BoundedWindow.TIMESTAMP_MAX_VALUE);
      if (value.getTimestamp().isBefore(minStamp)) {
        minStamp = value.getTimestamp();
        minStampState.write(minStamp);
        setupFlushTimerAndWatermarkHold(namespace, window, minStamp);
      }
    } else {
      reportDroppedElement(value, window);
    }
  }
 

private void onSortFlushTimer(BoundedWindow window, Instant timestamp) {
  StateInternals stateInternals = stepContext.stateInternals();
  StateNamespace namespace = StateNamespaces.window(windowCoder, window);
  BagState<WindowedValue<InputT>> sortBuffer = stateInternals.state(namespace, sortBufferTag);
  ValueState<Instant> minStampState = stateInternals.state(namespace, sortBufferMinStampTag);
  List<WindowedValue<InputT>> keep = new ArrayList<>();
  List<WindowedValue<InputT>> flush = new ArrayList<>();
  Instant newMinStamp = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<InputT> e : sortBuffer.read()) {
    if (!e.getTimestamp().isAfter(timestamp)) {
      flush.add(e);
    } else {
      keep.add(e);
      if (e.getTimestamp().isBefore(newMinStamp)) {
        newMinStamp = e.getTimestamp();
      }
    }
  }
  flush.stream()
      .sorted(Comparator.comparing(WindowedValue::getTimestamp))
      .forEachOrdered(e -> processElementUnordered(window, e));
  sortBuffer.clear();
  keep.forEach(sortBuffer::add);
  minStampState.write(newMinStamp);
  if (newMinStamp.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
    setupFlushTimerAndWatermarkHold(namespace, window, newMinStamp);
  } else {
    clearWatermarkHold(namespace);
  }
}
 

@Override
@SuppressWarnings("unchecked")
public WindowedValue<T> call(WindowedValue<T> windowedValue) throws Exception {
  final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows());
  final T element = windowedValue.getValue();
  final Instant timestamp = windowedValue.getTimestamp();
  Collection<W> windows =
      ((WindowFn<T, W>) fn)
          .assignWindows(
              ((WindowFn<T, W>) fn).new AssignContext() {
                @Override
                public T element() {
                  return element;
                }

                @Override
                public Instant timestamp() {
                  return timestamp;
                }

                @Override
                public BoundedWindow window() {
                  return boundedWindow;
                }
              });
  return WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING);
}
 

SingleWindowWindowedAccumulator(
    Function<InputT, ValueT> toValue, WindowedValue<AccumT> accumulator) {
  this.toValue = toValue;
  this.windowAccumulator = accumulator.getValue();
  this.accTimestamp =
      accumulator.getTimestamp().equals(BoundedWindow.TIMESTAMP_MIN_VALUE)
          ? null
          : accumulator.getTimestamp();
  this.accWindow = getWindow(accumulator);
}
 

/** Passes the initial element/restriction pair. */
public void seed(WindowedValue<KV<InputT, RestrictionT>> elementRestriction) {
  initState(
      StateNamespaces.window(
          windowCoder, Iterables.getOnlyElement(elementRestriction.getWindows())));
  seedState.write(elementRestriction);
  inputTimestamp = elementRestriction.getTimestamp();
}
 

/**
 * Resumes from a timer and returns the current element/restriction pair (with an up-to-date value
 * of the restriction).
 */
public WindowedValue<KV<InputT, RestrictionT>> resume(TimerData timer) {
  initState(timer.getNamespace());
  WindowedValue<KV<InputT, RestrictionT>> seed = seedState.read();
  inputTimestamp = seed.getTimestamp();
  return seed.withValue(KV.of(seed.getValue().getKey(), restrictionState.read()));
}
 

@Override
public void processElement(WindowedValue<TestStreamIndex<T>> element) throws Exception {
  TestStreamIndex<T> streamIndex = element.getValue();
  List<Event<T>> events = streamIndex.getTestStream().getEvents();
  int index = streamIndex.getIndex();
  Instant watermark = element.getTimestamp();
  Event<T> event = events.get(index);

  if (event.getType().equals(EventType.ELEMENT)) {
    UncommittedBundle<T> bundle =
        context.createBundle(
            (PCollection<T>) Iterables.getOnlyElement(application.getOutputs().values()));
    for (TimestampedValue<T> elem : ((ElementEvent<T>) event).getElements()) {
      bundle.add(
          WindowedValue.timestampedValueInGlobalWindow(elem.getValue(), elem.getTimestamp()));
    }
    resultBuilder.addOutput(bundle);
  }

  if (event.getType().equals(EventType.WATERMARK)) {
    watermark = ((WatermarkEvent<T>) event).getWatermark();
  }

  if (event.getType().equals(EventType.PROCESSING_TIME)) {
    ((TestClock) context.getClock())
        .advance(((ProcessingTimeEvent<T>) event).getProcessingTimeAdvance());
  }

  TestStreamIndex<T> next = streamIndex.next();
  if (next.getIndex() < events.size()) {
    resultBuilder.addUnprocessedElements(
        Collections.singleton(WindowedValue.timestampedValueInGlobalWindow(next, watermark)));
  }
}
 

private static Instant minTimestamp(Iterable<? extends WindowedValue<?>> elements) {
  Instant minTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
  for (WindowedValue<?> element : elements) {
    if (element.getTimestamp().isBefore(minTs)) {
      minTs = element.getTimestamp();
    }
  }
  return minTs;
}
 

@Override
public Instant getKey(WindowedValue<KV<K, V>> in) throws Exception {
  return in.getTimestamp();
}