Java Code Examples for org.apache.beam.sdk.util.WindowedValue#of()

The following examples show how to use org.apache.beam.sdk.util.WindowedValue#of() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapToTupleFunction.java    From twister2 with Apache License 2.0 6 votes vote down vote up
@Override
public Tuple<byte[], byte[]> map(WindowedValue<KV<K, V>> input) {
  Tuple<byte[], byte[]> element = null;

  WindowedValue<KV<K, WindowedValue<V>>> temp =
      WindowedValue.of(
          KV.of(
              input.getValue().getKey(),
              WindowedValue.of(
                  input.getValue().getValue(),
                  input.getTimestamp(),
                  input.getWindows(),
                  input.getPane())),
          input.getTimestamp(),
          input.getWindows(),
          input.getPane());
  try {
    element =
        new Tuple<>(
            CoderUtils.encodeToByteArray(keyCoder, temp.getValue().getKey()),
            CoderUtils.encodeToByteArray(wvCoder, temp.getValue().getValue()));
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return element;
}
 
Example 2
Source File: ImmutabilityCheckingBundleFactoryTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void mutationAfterAddKeyedBundleThrows() {
  UncommittedBundle<byte[]> keyed =
      factory.createKeyedBundle(StructuralKey.of("mykey", StringUtf8Coder.of()), transformed);

  byte[] array = new byte[] {4, 8, 12};
  WindowedValue<byte[]> windowedArray =
      WindowedValue.of(
          array,
          new Instant(891L),
          new IntervalWindow(new Instant(0), new Instant(1000)),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  keyed.add(windowedArray);

  array[0] = Byte.MAX_VALUE;
  thrown.expect(IllegalMutationException.class);
  thrown.expectMessage("Values must not be mutated in any way after being output");
  keyed.commit(Instant.now());
}
 
Example 3
Source File: ImmutabilityCheckingBundleFactoryTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void mutationBeforeAddCreateBundleSucceeds() {
  UncommittedBundle<byte[]> intermediate = factory.createBundle(transformed);

  byte[] array = new byte[] {4, 8, 12};
  WindowedValue<byte[]> windowedArray =
      WindowedValue.of(
          array,
          new Instant(891L),
          new IntervalWindow(new Instant(0), new Instant(1000)),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  array[2] = -3;
  intermediate.add(windowedArray);

  CommittedBundle<byte[]> committed = intermediate.commit(Instant.now());
  assertThat(committed.getElements(), containsInAnyOrder(windowedArray));
}
 
Example 4
Source File: SimplePushbackSideInputDoFnRunnerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void processElementNoSideInputs() {
  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of());

  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(
              new IntervalWindow(new Instant(-500L), new Instant(0L)),
              new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)),
              GlobalWindow.INSTANCE),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(multiWindowPushback, emptyIterable());
  // Should preserve the compressed representation when there's no side inputs.
  assertThat(underlying.inputElems, containsInAnyOrder(multiWindow));
}
 
Example 5
Source File: WindowDoFnOperatorTest.java    From beam with Apache License 2.0 5 votes vote down vote up
StreamRecord<WindowedValue<KeyedWorkItem<Long, Long>>> toStreamRecord() {
  WindowedValue<Long> item = WindowedValue.of(value, new Instant(timestamp), window, NO_FIRING);
  WindowedValue<KeyedWorkItem<Long, Long>> keyedItem =
      WindowedValue.of(
          new SingletonKeyedWorkItem<>(key, item), new Instant(timestamp), window, NO_FIRING);
  return new StreamRecord<>(keyedItem);
}
 
Example 6
Source File: TranslationUtils.java    From beam with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public WindowedValue<KV<K, OutputT>> call(WindowedValue<KV<K, Iterable<InputT>>> windowedKv)
    throws Exception {
  return WindowedValue.of(
      KV.of(
          windowedKv.getValue().getKey(),
          fn.getCombineFn()
              .apply(windowedKv.getValue().getValue(), fn.ctxtForValue(windowedKv))),
      windowedKv.getTimestamp(),
      windowedKv.getWindows(),
      windowedKv.getPane());
}
 
Example 7
Source File: SparkAssignWindowFn.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public WindowedValue<T> call(WindowedValue<T> windowedValue) throws Exception {
  final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows());
  final T element = windowedValue.getValue();
  final Instant timestamp = windowedValue.getTimestamp();
  Collection<W> windows =
      ((WindowFn<T, W>) fn)
          .assignWindows(
              ((WindowFn<T, W>) fn).new AssignContext() {
                @Override
                public T element() {
                  return element;
                }

                @Override
                public Instant timestamp() {
                  return timestamp;
                }

                @Override
                public BoundedWindow window() {
                  return boundedWindow;
                }
              });
  return WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING);
}
 
Example 8
Source File: PartialGroupByKeyParDoFns.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public void processElement(Object elem) throws Exception {
  @SuppressWarnings({"unchecked"})
  WindowedValue<KV<K, InputT>> input = (WindowedValue<KV<K, InputT>>) elem;
  for (BoundedWindow w : input.getWindows()) {
    WindowedValue<KV<K, InputT>> windowsExpandedInput =
        WindowedValue.of(input.getValue(), input.getTimestamp(), w, input.getPane());
    groupingTable.put(windowsExpandedInput, receiver);
  }
}
 
Example 9
Source File: SourceInputFormat.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public WindowedValue<T> nextRecord(WindowedValue<T> t) throws IOException {
  if (inputAvailable) {
    final T current = reader.getCurrent();
    final Instant timestamp = reader.getCurrentTimestamp();
    // advance reader to have a record ready next time
    inputAvailable = readerInvoker.invokeAdvance(reader);
    return WindowedValue.of(current, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
  }

  return null;
}
 
Example 10
Source File: ImmutableListBundleFactoryTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void withElementsShouldReturnIndependentBundle() {
  WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1);
  WindowedValue<Integer> secondValue =
      WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L));

  CommittedBundle<Integer> committed =
      afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue));

  WindowedValue<Integer> firstReplacement =
      WindowedValue.of(
          9,
          new Instant(2048L),
          new IntervalWindow(new Instant(2044L), Instant.now()),
          PaneInfo.NO_FIRING);
  WindowedValue<Integer> secondReplacement =
      WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now());
  CommittedBundle<Integer> withed =
      committed.withElements(ImmutableList.of(firstReplacement, secondReplacement));

  assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement));
  assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue));
  assertThat(withed.getKey(), equalTo(committed.getKey()));
  assertThat(withed.getPCollection(), equalTo(committed.getPCollection()));
  assertThat(
      withed.getSynchronizedProcessingOutputWatermark(),
      equalTo(committed.getSynchronizedProcessingOutputWatermark()));
  assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L)));
}
 
Example 11
Source File: TestStreamSource.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public void run(SourceContext<WindowedValue<T>> ctx) throws CoderException {
  TestStream<T> testStream = testStreamDecoder.apply(payload);
  List<TestStream.Event<T>> events = testStream.getEvents();

  for (int eventId = 0; isRunning && eventId < events.size(); eventId++) {
    TestStream.Event<T> event = events.get(eventId);

    synchronized (ctx.getCheckpointLock()) {
      if (event instanceof TestStream.ElementEvent) {
        for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) {
          Instant timestamp = element.getTimestamp();
          WindowedValue<T> value =
              WindowedValue.of(
                  element.getValue(), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
          ctx.collectWithTimestamp(value, timestamp.getMillis());
        }
      } else if (event instanceof TestStream.WatermarkEvent) {
        long millis = ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis();
        ctx.emitWatermark(new Watermark(millis));
      } else if (event instanceof TestStream.ProcessingTimeEvent) {
        // There seems to be no clean way to implement this
        throw new UnsupportedOperationException(
            "Advancing Processing time is not supported by the Flink Runner.");
      } else {
        throw new IllegalStateException("Unknown event type " + event);
      }
    }
  }
}
 
Example 12
Source File: FlinkStreamingTransformTranslators.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public void collectWithTimestamp(
    WindowedValue<ValueWithRecordId<OutputT>> element, long timestamp) {
  OutputT originalValue = element.getValue().getValue();
  WindowedValue<OutputT> output =
      WindowedValue.of(
          originalValue, element.getTimestamp(), element.getWindows(), element.getPane());
  ctx.collectWithTimestamp(output, timestamp);
}
 
Example 13
Source File: FlinkStreamingTransformTranslators.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public void collect(WindowedValue<ValueWithRecordId<OutputT>> element) {
  OutputT originalValue = element.getValue().getValue();
  WindowedValue<OutputT> output =
      WindowedValue.of(
          originalValue, element.getTimestamp(), element.getWindows(), element.getPane());
  ctx.collect(output);
}
 
Example 14
Source File: DataflowProcessFnRunner.java    From beam with Apache License 2.0 5 votes vote down vote up
private static <T> WindowedValue<KeyedWorkItem<byte[], T>> placeIntoElementWindow(
    WindowedValue<KeyedWorkItem<byte[], T>> compressedElem) {
  checkTrivialOuterWindows(compressedElem);
  WindowedValue<KeyedWorkItem<byte[], T>> res =
      WindowedValue.of(
          compressedElem.getValue(),
          BoundedWindow.TIMESTAMP_MIN_VALUE,
          getUnderlyingWindow(compressedElem.getValue()),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  return res;
}
 
Example 15
Source File: FnApiDoFnRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
private <T> WindowedValue<T> valueInWindows(
    T value, BoundedWindow window, BoundedWindow... windows) {
  return WindowedValue.of(
      value,
      window.maxTimestamp(),
      ImmutableList.<BoundedWindow>builder().add(window).add(windows).build(),
      PaneInfo.NO_FIRING);
}
 
Example 16
Source File: AssignWindowsParDoFnFactory.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void processElement(Object untypedElem) throws Exception {
  WindowedValue<T> elem = (WindowedValue<T>) untypedElem;

  Collection<W> windows =
      windowFn.assignWindows(
          windowFn.new AssignContext() {
            @Override
            public T element() {
              return elem.getValue();
            }

            @Override
            public Instant timestamp() {
              return elem.getTimestamp();
            }

            @Override
            public BoundedWindow window() {
              return Iterables.getOnlyElement(elem.getWindows());
            }
          });

  WindowedValue<T> res =
      WindowedValue.of(elem.getValue(), elem.getTimestamp(), windows, elem.getPane());
  receiver.process(res);
}
 
Example 17
Source File: ValueAndCoderLazySerializableTest.java    From beam with Apache License 2.0 4 votes vote down vote up
private <T> WindowedValue<T> winVal(T val) {
  return WindowedValue.of(val, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
}
 
Example 18
Source File: StreamingSideInputDoFnRunnerTest.java    From beam with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultipleWindowsNotReady() throws Exception {
  PCollectionView<String> view = createView();

  when(stepContext.getSideInputNotifications())
      .thenReturn(Arrays.<Windmill.GlobalDataId>asList());
  when(stepContext.issueSideInputFetch(
          eq(view), any(BoundedWindow.class), eq(SideInputState.UNKNOWN)))
      .thenReturn(false);

  ListOutputManager outputManager = new ListOutputManager();

  List<PCollectionView<String>> views = Arrays.asList(view);
  StreamingSideInputFetcher<String, IntervalWindow> sideInputFetcher = createFetcher(views);
  StreamingSideInputDoFnRunner<String, String, IntervalWindow> runner =
      createRunner(
          SlidingWindows.of(Duration.millis(10)).every(Duration.millis(10)),
          outputManager,
          views,
          sideInputFetcher);

  IntervalWindow window1 = new IntervalWindow(new Instant(0), new Instant(10));
  IntervalWindow window2 = new IntervalWindow(new Instant(-5), new Instant(5));
  long timestamp = 1L;

  WindowedValue<String> elem =
      WindowedValue.of(
          "e", new Instant(timestamp), Arrays.asList(window1, window2), PaneInfo.NO_FIRING);

  runner.startBundle();
  runner.processElement(elem);
  runner.finishBundle();

  assertTrue(outputManager.getOutput(mainOutputTag).isEmpty());

  // Verify that we added the element to an appropriate tag list, and that we buffered the element
  // in both windows separately
  ValueState<Map<IntervalWindow, Set<GlobalDataRequest>>> blockedMapState =
      state.state(
          StateNamespaces.global(),
          StreamingSideInputFetcher.blockedMapAddr(WINDOW_FN.windowCoder()));

  Map<IntervalWindow, Set<GlobalDataRequest>> blockedMap = blockedMapState.read();

  assertThat(
      blockedMap.get(window1),
      equalTo(
          Collections.singleton(
              Windmill.GlobalDataRequest.newBuilder()
                  .setDataId(
                      Windmill.GlobalDataId.newBuilder()
                          .setTag(view.getTagInternal().getId())
                          .setVersion(
                              ByteString.copyFrom(
                                  CoderUtils.encodeToByteArray(
                                      IntervalWindow.getCoder(), window1)))
                          .build())
                  .setExistenceWatermarkDeadline(9000)
                  .build())));

  assertThat(
      blockedMap.get(window2),
      equalTo(
          Collections.singleton(
              Windmill.GlobalDataRequest.newBuilder()
                  .setDataId(
                      Windmill.GlobalDataId.newBuilder()
                          .setTag(view.getTagInternal().getId())
                          .setVersion(
                              ByteString.copyFrom(
                                  CoderUtils.encodeToByteArray(
                                      IntervalWindow.getCoder(), window1)))
                          .build())
                  .setExistenceWatermarkDeadline(9000)
                  .build())));

  assertThat(
      sideInputFetcher.elementBag(window1).read(),
      contains(Iterables.get(elem.explodeWindows(), 0)));

  assertThat(
      sideInputFetcher.elementBag(window2).read(),
      contains(Iterables.get(elem.explodeWindows(), 1)));

  assertEquals(sideInputFetcher.watermarkHold(window1).read(), new Instant(timestamp));
  assertEquals(sideInputFetcher.watermarkHold(window2).read(), new Instant(timestamp));
}
 
Example 19
Source File: LateDataDroppingDoFnRunnerTest.java    From beam with Apache License 2.0 4 votes vote down vote up
private <T> WindowedValue<T> createDatum(T element, long timestampMillis) {
  Instant timestamp = new Instant(timestampMillis);
  return WindowedValue.of(
      element, timestamp, Arrays.asList(WINDOW_FN.assignWindow(timestamp)), PaneInfo.NO_FIRING);
}
 
Example 20
Source File: WindowGroupP.java    From beam with Apache License 2.0 4 votes vote down vote up
KeyManager(K key) {
  this.timerInternals = new InMemoryTimerInternals();
  this.stateInternals = new InMemoryStateInternalsImpl(key);
  this.reduceFnRunner =
      new ReduceFnRunner<>(
          key,
          windowingStrategy,
          ExecutableTriggerStateMachine.create(
              TriggerStateMachines.stateMachineForTrigger(
                  TriggerTranslation.toProto(windowingStrategy.getTrigger()))),
          stateInternals,
          timerInternals,
          new OutputWindowedValue<KV<K, Iterable<V>>>() {
            @Override
            public void outputWindowedValue(
                KV<K, Iterable<V>> output,
                Instant timestamp,
                Collection<? extends BoundedWindow> windows,
                PaneInfo pane) {
              WindowedValue<KV<K, Iterable<V>>> windowedValue =
                  WindowedValue.of(output, timestamp, windows, pane);
              byte[] encodedValue = Utils.encode(windowedValue, outputCoder);
              //noinspection ResultOfMethodCallIgnored
              appendableTraverser.append(encodedValue);
            }

            @Override
            public <AdditionalOutputT> void outputWindowedValue(
                TupleTag<AdditionalOutputT> tag,
                AdditionalOutputT output,
                Instant timestamp,
                Collection<? extends BoundedWindow> windows,
                PaneInfo pane) {
              throw new UnsupportedOperationException("Grouping should not use side outputs");
            }
          },
          NullSideInputReader.empty(),
          SystemReduceFn.buffering(inputValueValueCoder),
          pipelineOptions.get());
  advanceWatermark(latestWatermark, Instant.now());
}