Java Code Examples for org.apache.beam.sdk.util.WindowedValue#of()

@Override
public Tuple<byte[], byte[]> map(WindowedValue<KV<K, V>> input) {
  Tuple<byte[], byte[]> element = null;

  WindowedValue<KV<K, WindowedValue<V>>> temp =
      WindowedValue.of(
          KV.of(
              input.getValue().getKey(),
              WindowedValue.of(
                  input.getValue().getValue(),
                  input.getTimestamp(),
                  input.getWindows(),
                  input.getPane())),
          input.getTimestamp(),
          input.getWindows(),
          input.getPane());
  try {
    element =
        new Tuple<>(
            CoderUtils.encodeToByteArray(keyCoder, temp.getValue().getKey()),
            CoderUtils.encodeToByteArray(wvCoder, temp.getValue().getValue()));
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return element;
}
 

@Test
public void mutationAfterAddKeyedBundleThrows() {
  UncommittedBundle<byte[]> keyed =
      factory.createKeyedBundle(StructuralKey.of("mykey", StringUtf8Coder.of()), transformed);

  byte[] array = new byte[] {4, 8, 12};
  WindowedValue<byte[]> windowedArray =
      WindowedValue.of(
          array,
          new Instant(891L),
          new IntervalWindow(new Instant(0), new Instant(1000)),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  keyed.add(windowedArray);

  array[0] = Byte.MAX_VALUE;
  thrown.expect(IllegalMutationException.class);
  thrown.expectMessage("Values must not be mutated in any way after being output");
  keyed.commit(Instant.now());
}
 

@Test
public void mutationBeforeAddCreateBundleSucceeds() {
  UncommittedBundle<byte[]> intermediate = factory.createBundle(transformed);

  byte[] array = new byte[] {4, 8, 12};
  WindowedValue<byte[]> windowedArray =
      WindowedValue.of(
          array,
          new Instant(891L),
          new IntervalWindow(new Instant(0), new Instant(1000)),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  array[2] = -3;
  intermediate.add(windowedArray);

  CommittedBundle<byte[]> committed = intermediate.commit(Instant.now());
  assertThat(committed.getElements(), containsInAnyOrder(windowedArray));
}
 

@Test
public void processElementNoSideInputs() {
  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of());

  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(
              new IntervalWindow(new Instant(-500L), new Instant(0L)),
              new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)),
              GlobalWindow.INSTANCE),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(multiWindowPushback, emptyIterable());
  // Should preserve the compressed representation when there's no side inputs.
  assertThat(underlying.inputElems, containsInAnyOrder(multiWindow));
}
 

StreamRecord<WindowedValue<KeyedWorkItem<Long, Long>>> toStreamRecord() {
  WindowedValue<Long> item = WindowedValue.of(value, new Instant(timestamp), window, NO_FIRING);
  WindowedValue<KeyedWorkItem<Long, Long>> keyedItem =
      WindowedValue.of(
          new SingletonKeyedWorkItem<>(key, item), new Instant(timestamp), window, NO_FIRING);
  return new StreamRecord<>(keyedItem);
}
 

@SuppressWarnings("unchecked")
@Override
public WindowedValue<KV<K, OutputT>> call(WindowedValue<KV<K, Iterable<InputT>>> windowedKv)
    throws Exception {
  return WindowedValue.of(
      KV.of(
          windowedKv.getValue().getKey(),
          fn.getCombineFn()
              .apply(windowedKv.getValue().getValue(), fn.ctxtForValue(windowedKv))),
      windowedKv.getTimestamp(),
      windowedKv.getWindows(),
      windowedKv.getPane());
}
 

@Override
@SuppressWarnings("unchecked")
public WindowedValue<T> call(WindowedValue<T> windowedValue) throws Exception {
  final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows());
  final T element = windowedValue.getValue();
  final Instant timestamp = windowedValue.getTimestamp();
  Collection<W> windows =
      ((WindowFn<T, W>) fn)
          .assignWindows(
              ((WindowFn<T, W>) fn).new AssignContext() {
                @Override
                public T element() {
                  return element;
                }

                @Override
                public Instant timestamp() {
                  return timestamp;
                }

                @Override
                public BoundedWindow window() {
                  return boundedWindow;
                }
              });
  return WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING);
}
 

@Override
public void processElement(Object elem) throws Exception {
  @SuppressWarnings({"unchecked"})
  WindowedValue<KV<K, InputT>> input = (WindowedValue<KV<K, InputT>>) elem;
  for (BoundedWindow w : input.getWindows()) {
    WindowedValue<KV<K, InputT>> windowsExpandedInput =
        WindowedValue.of(input.getValue(), input.getTimestamp(), w, input.getPane());
    groupingTable.put(windowsExpandedInput, receiver);
  }
}
 

@Override
public WindowedValue<T> nextRecord(WindowedValue<T> t) throws IOException {
  if (inputAvailable) {
    final T current = reader.getCurrent();
    final Instant timestamp = reader.getCurrentTimestamp();
    // advance reader to have a record ready next time
    inputAvailable = readerInvoker.invokeAdvance(reader);
    return WindowedValue.of(current, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
  }

  return null;
}
 

@SuppressWarnings("unchecked")
@Test
public void withElementsShouldReturnIndependentBundle() {
  WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
  WindowedValue<Integer> secondValue =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));

  CommittedBundle<Integer> committed =
      afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));

  WindowedValue<Integer> firstReplacement =
      WindowedValue.of(
          9,
          new Instant(2048L),
          new IntervalWindow(new Instant(2044L), Instant.now()),
          PaneInfo.NO_FIRING);
  WindowedValue<Integer> secondReplacement =
      WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now());
  CommittedBundle<Integer> withed =
      committed.withElements(ImmutableList.of(firstReplacement, secondReplacement));

  assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement));
  assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue));
  assertThat(withed.getKey(), equalTo(committed.getKey()));
  assertThat(withed.getPCollection(), equalTo(committed.getPCollection()));
  assertThat(
      withed.getSynchronizedProcessingOutputWatermark(),
      equalTo(committed.getSynchronizedProcessingOutputWatermark()));
  assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L)));
}
 

@Override
public void run(SourceContext<WindowedValue<T>> ctx) throws CoderException {
  TestStream<T> testStream = testStreamDecoder.apply(payload);
  List<TestStream.Event<T>> events = testStream.getEvents();

  for (int eventId = 0; isRunning && eventId < events.size(); eventId++) {
    TestStream.Event<T> event = events.get(eventId);

    synchronized (ctx.getCheckpointLock()) {
      if (event instanceof TestStream.ElementEvent) {
        for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) {
          Instant timestamp = element.getTimestamp();
          WindowedValue<T> value =
              WindowedValue.of(
                  element.getValue(), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
          ctx.collectWithTimestamp(value, timestamp.getMillis());
        }
      } else if (event instanceof TestStream.WatermarkEvent) {
        long millis = ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis();
        ctx.emitWatermark(new Watermark(millis));
      } else if (event instanceof TestStream.ProcessingTimeEvent) {
        // There seems to be no clean way to implement this
        throw new UnsupportedOperationException(
            "Advancing Processing time is not supported by the Flink Runner.");
      } else {
        throw new IllegalStateException("Unknown event type " + event);
      }
    }
  }
}
 

@Override
public void collectWithTimestamp(
    WindowedValue<ValueWithRecordId<OutputT>> element, long timestamp) {
  OutputT originalValue = element.getValue().getValue();
  WindowedValue<OutputT> output =
      WindowedValue.of(
          originalValue, element.getTimestamp(), element.getWindows(), element.getPane());
  ctx.collectWithTimestamp(output, timestamp);
}
 

@Override
public void collect(WindowedValue<ValueWithRecordId<OutputT>> element) {
  OutputT originalValue = element.getValue().getValue();
  WindowedValue<OutputT> output =
      WindowedValue.of(
          originalValue, element.getTimestamp(), element.getWindows(), element.getPane());
  ctx.collect(output);
}
 

private static <T> WindowedValue<KeyedWorkItem<byte[], T>> placeIntoElementWindow(
    WindowedValue<KeyedWorkItem<byte[], T>> compressedElem) {
  checkTrivialOuterWindows(compressedElem);
  WindowedValue<KeyedWorkItem<byte[], T>> res =
      WindowedValue.of(
          compressedElem.getValue(),
          BoundedWindow.TIMESTAMP_MIN_VALUE,
          getUnderlyingWindow(compressedElem.getValue()),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  return res;
}
 

private <T> WindowedValue<T> valueInWindows(
    T value, BoundedWindow window, BoundedWindow... windows) {
  return WindowedValue.of(
      value,
      window.maxTimestamp(),
      ImmutableList.<BoundedWindow>builder().add(window).add(windows).build(),
      PaneInfo.NO_FIRING);
}
 

@Override
@SuppressWarnings("unchecked")
public void processElement(Object untypedElem) throws Exception {
  WindowedValue<T> elem = (WindowedValue<T>) untypedElem;

  Collection<W> windows =
      windowFn.assignWindows(
          windowFn.new AssignContext() {
            @Override
            public T element() {
              return elem.getValue();
            }

            @Override
            public Instant timestamp() {
              return elem.getTimestamp();
            }

            @Override
            public BoundedWindow window() {
              return Iterables.getOnlyElement(elem.getWindows());
            }
          });

  WindowedValue<T> res =
      WindowedValue.of(elem.getValue(), elem.getTimestamp(), windows, elem.getPane());
  receiver.process(res);
}
 

private <T> WindowedValue<T> winVal(T val) {
  return WindowedValue.of(val, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
}
 

@Test
public void testMultipleWindowsNotReady() throws Exception {
  PCollectionView<String> view = createView();

  when(stepContext.getSideInputNotifications())
      .thenReturn(Arrays.<Windmill.GlobalDataId>asList());
  when(stepContext.issueSideInputFetch(
          eq(view), any(BoundedWindow.class), eq(SideInputState.UNKNOWN)))
      .thenReturn(false);

  ListOutputManager outputManager = new ListOutputManager();

  List<PCollectionView<String>> views = Arrays.asList(view);
  StreamingSideInputFetcher<String, IntervalWindow> sideInputFetcher = createFetcher(views);
  StreamingSideInputDoFnRunner<String, String, IntervalWindow> runner =
      createRunner(
          SlidingWindows.of(Duration.millis(10)).every(Duration.millis(10)),
          outputManager,
          views,
          sideInputFetcher);

  IntervalWindow window1 = new IntervalWindow(new Instant(0), new Instant(10));
  IntervalWindow window2 = new IntervalWindow(new Instant(-5), new Instant(5));
  long timestamp = 1L;

  WindowedValue<String> elem =
      WindowedValue.of(
          "e", new Instant(timestamp), Arrays.asList(window1, window2), PaneInfo.NO_FIRING);

  runner.startBundle();
  runner.processElement(elem);
  runner.finishBundle();

  assertTrue(outputManager.getOutput(mainOutputTag).isEmpty());

  // Verify that we added the element to an appropriate tag list, and that we buffered the element
  // in both windows separately
  ValueState<Map<IntervalWindow, Set<GlobalDataRequest>>> blockedMapState =
      state.state(
          StateNamespaces.global(),
          StreamingSideInputFetcher.blockedMapAddr(WINDOW_FN.windowCoder()));

  Map<IntervalWindow, Set<GlobalDataRequest>> blockedMap = blockedMapState.read();

  assertThat(
      blockedMap.get(window1),
      equalTo(
          Collections.singleton(
              Windmill.GlobalDataRequest.newBuilder()
                  .setDataId(
                      Windmill.GlobalDataId.newBuilder()
                          .setTag(view.getTagInternal().getId())
                          .setVersion(
                              ByteString.copyFrom(
                                  CoderUtils.encodeToByteArray(
                                      IntervalWindow.getCoder(), window1)))
                          .build())
                  .setExistenceWatermarkDeadline(9000)
                  .build())));

  assertThat(
      blockedMap.get(window2),
      equalTo(
          Collections.singleton(
              Windmill.GlobalDataRequest.newBuilder()
                  .setDataId(
                      Windmill.GlobalDataId.newBuilder()
                          .setTag(view.getTagInternal().getId())
                          .setVersion(
                              ByteString.copyFrom(
                                  CoderUtils.encodeToByteArray(
                                      IntervalWindow.getCoder(), window1)))
                          .build())
                  .setExistenceWatermarkDeadline(9000)
                  .build())));

  assertThat(
      sideInputFetcher.elementBag(window1).read(),
      contains(Iterables.get(elem.explodeWindows(), 0)));

  assertThat(
      sideInputFetcher.elementBag(window2).read(),
      contains(Iterables.get(elem.explodeWindows(), 1)));

  assertEquals(sideInputFetcher.watermarkHold(window1).read(), new Instant(timestamp));
  assertEquals(sideInputFetcher.watermarkHold(window2).read(), new Instant(timestamp));
}
 

private <T> WindowedValue<T> createDatum(T element, long timestampMillis) {
  Instant timestamp = new Instant(timestampMillis);
  return WindowedValue.of(
      element, timestamp, Arrays.asList(WINDOW_FN.assignWindow(timestamp)), PaneInfo.NO_FIRING);
}
 

KeyManager(K key) {
  this.timerInternals = new InMemoryTimerInternals();
  this.stateInternals = new InMemoryStateInternalsImpl(key);
  this.reduceFnRunner =
      new ReduceFnRunner<>(
          key,
          windowingStrategy,
          ExecutableTriggerStateMachine.create(
              TriggerStateMachines.stateMachineForTrigger(
                  TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
          stateInternals,
          timerInternals,
          new OutputWindowedValue<KV<K, Iterable<V>>>() {
            @Override
            public void outputWindowedValue(
                KV<K, Iterable<V>> output,
                Instant timestamp,
                Collection<? extends BoundedWindow> windows,
                PaneInfo pane) {
              WindowedValue<KV<K, Iterable<V>>> windowedValue =
                  WindowedValue.of(output, timestamp, windows, pane);
              byte[] encodedValue = Utils.encode(windowedValue, outputCoder);
              //noinspection ResultOfMethodCallIgnored
              appendableTraverser.append(encodedValue);
            }

            @Override
            public <AdditionalOutputT> void outputWindowedValue(
                TupleTag<AdditionalOutputT> tag,
                AdditionalOutputT output,
                Instant timestamp,
                Collection<? extends BoundedWindow> windows,
                PaneInfo pane) {
              throw new UnsupportedOperationException("Grouping should not use side outputs");
            }
          },
          NullSideInputReader.empty(),
          SystemReduceFn.buffering(inputValueValueCoder),
          pipelineOptions.get());
  advanceWatermark(latestWatermark, Instant.now());
}