Java Code Examples for org.apache.beam.sdk.util.WindowedValue#of()
The following examples show how to use
org.apache.beam.sdk.util.WindowedValue#of() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapToTupleFunction.java From twister2 with Apache License 2.0 | 6 votes |
@Override public Tuple<byte[], byte[]> map(WindowedValue<KV<K, V>> input) { Tuple<byte[], byte[]> element = null; WindowedValue<KV<K, WindowedValue<V>>> temp = WindowedValue.of( KV.of( input.getValue().getKey(), WindowedValue.of( input.getValue().getValue(), input.getTimestamp(), input.getWindows(), input.getPane())), input.getTimestamp(), input.getWindows(), input.getPane()); try { element = new Tuple<>( CoderUtils.encodeToByteArray(keyCoder, temp.getValue().getKey()), CoderUtils.encodeToByteArray(wvCoder, temp.getValue().getValue())); } catch (CoderException e) { LOG.info(e.getMessage()); } return element; }
Example 2
Source File: ImmutabilityCheckingBundleFactoryTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void mutationAfterAddKeyedBundleThrows() { UncommittedBundle<byte[]> keyed = factory.createKeyedBundle(StructuralKey.of("mykey", StringUtf8Coder.of()), transformed); byte[] array = new byte[] {4, 8, 12}; WindowedValue<byte[]> windowedArray = WindowedValue.of( array, new Instant(891L), new IntervalWindow(new Instant(0), new Instant(1000)), PaneInfo.ON_TIME_AND_ONLY_FIRING); keyed.add(windowedArray); array[0] = Byte.MAX_VALUE; thrown.expect(IllegalMutationException.class); thrown.expectMessage("Values must not be mutated in any way after being output"); keyed.commit(Instant.now()); }
Example 3
Source File: ImmutabilityCheckingBundleFactoryTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void mutationBeforeAddCreateBundleSucceeds() { UncommittedBundle<byte[]> intermediate = factory.createBundle(transformed); byte[] array = new byte[] {4, 8, 12}; WindowedValue<byte[]> windowedArray = WindowedValue.of( array, new Instant(891L), new IntervalWindow(new Instant(0), new Instant(1000)), PaneInfo.ON_TIME_AND_ONLY_FIRING); array[2] = -3; intermediate.add(windowedArray); CommittedBundle<byte[]> committed = intermediate.commit(Instant.now()); assertThat(committed.getElements(), containsInAnyOrder(windowedArray)); }
Example 4
Source File: SimplePushbackSideInputDoFnRunnerTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void processElementNoSideInputs() { SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of()); WindowedValue<Integer> multiWindow = WindowedValue.of( 2, new Instant(-2), ImmutableList.of( new IntervalWindow(new Instant(-500L), new Instant(0L)), new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)), GlobalWindow.INSTANCE), PaneInfo.ON_TIME_AND_ONLY_FIRING); Iterable<WindowedValue<Integer>> multiWindowPushback = runner.processElementInReadyWindows(multiWindow); assertThat(multiWindowPushback, emptyIterable()); // Should preserve the compressed representation when there's no side inputs. assertThat(underlying.inputElems, containsInAnyOrder(multiWindow)); }
Example 5
Source File: WindowDoFnOperatorTest.java From beam with Apache License 2.0 | 5 votes |
StreamRecord<WindowedValue<KeyedWorkItem<Long, Long>>> toStreamRecord() { WindowedValue<Long> item = WindowedValue.of(value, new Instant(timestamp), window, NO_FIRING); WindowedValue<KeyedWorkItem<Long, Long>> keyedItem = WindowedValue.of( new SingletonKeyedWorkItem<>(key, item), new Instant(timestamp), window, NO_FIRING); return new StreamRecord<>(keyedItem); }
Example 6
Source File: TranslationUtils.java From beam with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public WindowedValue<KV<K, OutputT>> call(WindowedValue<KV<K, Iterable<InputT>>> windowedKv) throws Exception { return WindowedValue.of( KV.of( windowedKv.getValue().getKey(), fn.getCombineFn() .apply(windowedKv.getValue().getValue(), fn.ctxtForValue(windowedKv))), windowedKv.getTimestamp(), windowedKv.getWindows(), windowedKv.getPane()); }
Example 7
Source File: SparkAssignWindowFn.java From beam with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public WindowedValue<T> call(WindowedValue<T> windowedValue) throws Exception { final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows()); final T element = windowedValue.getValue(); final Instant timestamp = windowedValue.getTimestamp(); Collection<W> windows = ((WindowFn<T, W>) fn) .assignWindows( ((WindowFn<T, W>) fn).new AssignContext() { @Override public T element() { return element; } @Override public Instant timestamp() { return timestamp; } @Override public BoundedWindow window() { return boundedWindow; } }); return WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING); }
Example 8
Source File: PartialGroupByKeyParDoFns.java From beam with Apache License 2.0 | 5 votes |
@Override public void processElement(Object elem) throws Exception { @SuppressWarnings({"unchecked"}) WindowedValue<KV<K, InputT>> input = (WindowedValue<KV<K, InputT>>) elem; for (BoundedWindow w : input.getWindows()) { WindowedValue<KV<K, InputT>> windowsExpandedInput = WindowedValue.of(input.getValue(), input.getTimestamp(), w, input.getPane()); groupingTable.put(windowsExpandedInput, receiver); } }
Example 9
Source File: SourceInputFormat.java From beam with Apache License 2.0 | 5 votes |
@Override public WindowedValue<T> nextRecord(WindowedValue<T> t) throws IOException { if (inputAvailable) { final T current = reader.getCurrent(); final Instant timestamp = reader.getCurrentTimestamp(); // advance reader to have a record ready next time inputAvailable = readerInvoker.invokeAdvance(reader); return WindowedValue.of(current, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); } return null; }
Example 10
Source File: ImmutableListBundleFactoryTest.java From beam with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Test public void withElementsShouldReturnIndependentBundle() { WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> secondValue = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L)); CommittedBundle<Integer> committed = afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue)); WindowedValue<Integer> firstReplacement = WindowedValue.of( 9, new Instant(2048L), new IntervalWindow(new Instant(2044L), Instant.now()), PaneInfo.NO_FIRING); WindowedValue<Integer> secondReplacement = WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now()); CommittedBundle<Integer> withed = committed.withElements(ImmutableList.of(firstReplacement, secondReplacement)); assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement)); assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue)); assertThat(withed.getKey(), equalTo(committed.getKey())); assertThat(withed.getPCollection(), equalTo(committed.getPCollection())); assertThat( withed.getSynchronizedProcessingOutputWatermark(), equalTo(committed.getSynchronizedProcessingOutputWatermark())); assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L))); }
Example 11
Source File: TestStreamSource.java From beam with Apache License 2.0 | 5 votes |
@Override public void run(SourceContext<WindowedValue<T>> ctx) throws CoderException { TestStream<T> testStream = testStreamDecoder.apply(payload); List<TestStream.Event<T>> events = testStream.getEvents(); for (int eventId = 0; isRunning && eventId < events.size(); eventId++) { TestStream.Event<T> event = events.get(eventId); synchronized (ctx.getCheckpointLock()) { if (event instanceof TestStream.ElementEvent) { for (TimestampedValue<T> element : ((TestStream.ElementEvent<T>) event).getElements()) { Instant timestamp = element.getTimestamp(); WindowedValue<T> value = WindowedValue.of( element.getValue(), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); ctx.collectWithTimestamp(value, timestamp.getMillis()); } } else if (event instanceof TestStream.WatermarkEvent) { long millis = ((TestStream.WatermarkEvent<T>) event).getWatermark().getMillis(); ctx.emitWatermark(new Watermark(millis)); } else if (event instanceof TestStream.ProcessingTimeEvent) { // There seems to be no clean way to implement this throw new UnsupportedOperationException( "Advancing Processing time is not supported by the Flink Runner."); } else { throw new IllegalStateException("Unknown event type " + event); } } } }
Example 12
Source File: FlinkStreamingTransformTranslators.java From beam with Apache License 2.0 | 5 votes |
@Override public void collectWithTimestamp( WindowedValue<ValueWithRecordId<OutputT>> element, long timestamp) { OutputT originalValue = element.getValue().getValue(); WindowedValue<OutputT> output = WindowedValue.of( originalValue, element.getTimestamp(), element.getWindows(), element.getPane()); ctx.collectWithTimestamp(output, timestamp); }
Example 13
Source File: FlinkStreamingTransformTranslators.java From beam with Apache License 2.0 | 5 votes |
@Override public void collect(WindowedValue<ValueWithRecordId<OutputT>> element) { OutputT originalValue = element.getValue().getValue(); WindowedValue<OutputT> output = WindowedValue.of( originalValue, element.getTimestamp(), element.getWindows(), element.getPane()); ctx.collect(output); }
Example 14
Source File: DataflowProcessFnRunner.java From beam with Apache License 2.0 | 5 votes |
private static <T> WindowedValue<KeyedWorkItem<byte[], T>> placeIntoElementWindow( WindowedValue<KeyedWorkItem<byte[], T>> compressedElem) { checkTrivialOuterWindows(compressedElem); WindowedValue<KeyedWorkItem<byte[], T>> res = WindowedValue.of( compressedElem.getValue(), BoundedWindow.TIMESTAMP_MIN_VALUE, getUnderlyingWindow(compressedElem.getValue()), PaneInfo.ON_TIME_AND_ONLY_FIRING); return res; }
Example 15
Source File: FnApiDoFnRunnerTest.java From beam with Apache License 2.0 | 5 votes |
private <T> WindowedValue<T> valueInWindows( T value, BoundedWindow window, BoundedWindow... windows) { return WindowedValue.of( value, window.maxTimestamp(), ImmutableList.<BoundedWindow>builder().add(window).add(windows).build(), PaneInfo.NO_FIRING); }
Example 16
Source File: AssignWindowsParDoFnFactory.java From beam with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public void processElement(Object untypedElem) throws Exception { WindowedValue<T> elem = (WindowedValue<T>) untypedElem; Collection<W> windows = windowFn.assignWindows( windowFn.new AssignContext() { @Override public T element() { return elem.getValue(); } @Override public Instant timestamp() { return elem.getTimestamp(); } @Override public BoundedWindow window() { return Iterables.getOnlyElement(elem.getWindows()); } }); WindowedValue<T> res = WindowedValue.of(elem.getValue(), elem.getTimestamp(), windows, elem.getPane()); receiver.process(res); }
Example 17
Source File: ValueAndCoderLazySerializableTest.java From beam with Apache License 2.0 | 4 votes |
private <T> WindowedValue<T> winVal(T val) { return WindowedValue.of(val, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); }
Example 18
Source File: StreamingSideInputDoFnRunnerTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void testMultipleWindowsNotReady() throws Exception { PCollectionView<String> view = createView(); when(stepContext.getSideInputNotifications()) .thenReturn(Arrays.<Windmill.GlobalDataId>asList()); when(stepContext.issueSideInputFetch( eq(view), any(BoundedWindow.class), eq(SideInputState.UNKNOWN))) .thenReturn(false); ListOutputManager outputManager = new ListOutputManager(); List<PCollectionView<String>> views = Arrays.asList(view); StreamingSideInputFetcher<String, IntervalWindow> sideInputFetcher = createFetcher(views); StreamingSideInputDoFnRunner<String, String, IntervalWindow> runner = createRunner( SlidingWindows.of(Duration.millis(10)).every(Duration.millis(10)), outputManager, views, sideInputFetcher); IntervalWindow window1 = new IntervalWindow(new Instant(0), new Instant(10)); IntervalWindow window2 = new IntervalWindow(new Instant(-5), new Instant(5)); long timestamp = 1L; WindowedValue<String> elem = WindowedValue.of( "e", new Instant(timestamp), Arrays.asList(window1, window2), PaneInfo.NO_FIRING); runner.startBundle(); runner.processElement(elem); runner.finishBundle(); assertTrue(outputManager.getOutput(mainOutputTag).isEmpty()); // Verify that we added the element to an appropriate tag list, and that we buffered the element // in both windows separately ValueState<Map<IntervalWindow, Set<GlobalDataRequest>>> blockedMapState = state.state( StateNamespaces.global(), StreamingSideInputFetcher.blockedMapAddr(WINDOW_FN.windowCoder())); Map<IntervalWindow, Set<GlobalDataRequest>> blockedMap = blockedMapState.read(); assertThat( blockedMap.get(window1), equalTo( Collections.singleton( Windmill.GlobalDataRequest.newBuilder() .setDataId( Windmill.GlobalDataId.newBuilder() .setTag(view.getTagInternal().getId()) .setVersion( ByteString.copyFrom( CoderUtils.encodeToByteArray( IntervalWindow.getCoder(), window1))) .build()) .setExistenceWatermarkDeadline(9000) .build()))); assertThat( blockedMap.get(window2), equalTo( Collections.singleton( Windmill.GlobalDataRequest.newBuilder() .setDataId( Windmill.GlobalDataId.newBuilder() .setTag(view.getTagInternal().getId()) .setVersion( ByteString.copyFrom( CoderUtils.encodeToByteArray( IntervalWindow.getCoder(), window1))) .build()) .setExistenceWatermarkDeadline(9000) .build()))); assertThat( sideInputFetcher.elementBag(window1).read(), contains(Iterables.get(elem.explodeWindows(), 0))); assertThat( sideInputFetcher.elementBag(window2).read(), contains(Iterables.get(elem.explodeWindows(), 1))); assertEquals(sideInputFetcher.watermarkHold(window1).read(), new Instant(timestamp)); assertEquals(sideInputFetcher.watermarkHold(window2).read(), new Instant(timestamp)); }
Example 19
Source File: LateDataDroppingDoFnRunnerTest.java From beam with Apache License 2.0 | 4 votes |
private <T> WindowedValue<T> createDatum(T element, long timestampMillis) { Instant timestamp = new Instant(timestampMillis); return WindowedValue.of( element, timestamp, Arrays.asList(WINDOW_FN.assignWindow(timestamp)), PaneInfo.NO_FIRING); }
Example 20
Source File: WindowGroupP.java From beam with Apache License 2.0 | 4 votes |
KeyManager(K key) { this.timerInternals = new InMemoryTimerInternals(); this.stateInternals = new InMemoryStateInternalsImpl(key); this.reduceFnRunner = new ReduceFnRunner<>( key, windowingStrategy, ExecutableTriggerStateMachine.create( TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger()))), stateInternals, timerInternals, new OutputWindowedValue<KV<K, Iterable<V>>>() { @Override public void outputWindowedValue( KV<K, Iterable<V>> output, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) { WindowedValue<KV<K, Iterable<V>>> windowedValue = WindowedValue.of(output, timestamp, windows, pane); byte[] encodedValue = Utils.encode(windowedValue, outputCoder); //noinspection ResultOfMethodCallIgnored appendableTraverser.append(encodedValue); } @Override public <AdditionalOutputT> void outputWindowedValue( TupleTag<AdditionalOutputT> tag, AdditionalOutputT output, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) { throw new UnsupportedOperationException("Grouping should not use side outputs"); } }, NullSideInputReader.empty(), SystemReduceFn.buffering(inputValueValueCoder), pipelineOptions.get()); advanceWatermark(latestWatermark, Instant.now()); }