org.apache.beam.sdk.transforms.windowing.FixedWindows Java Examples

The following examples show how to use org.apache.beam.sdk.transforms.windowing.FixedWindows. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LateDataUtilsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(-100));
  assertThat(
      window.maxTimestamp().plus(allowedLateness),
      Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 
Example #2
Source File: GroupByKeyTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testIdentityWindowFnPropagation() {

  List<KV<String, Integer>> ungroupedPairs = Arrays.asList();

  PCollection<KV<String, Integer>> input =
      p.apply(
              Create.of(ungroupedPairs)
                  .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
          .apply(Window.into(FixedWindows.of(Duration.standardMinutes(1))));

  PCollection<KV<String, Iterable<Integer>>> output = input.apply(GroupByKey.create());

  p.run();

  Assert.assertTrue(
      output
          .getWindowingStrategy()
          .getWindowFn()
          .isCompatible(FixedWindows.of(Duration.standardMinutes(1))));
}
 
Example #3
Source File: AfterWatermarkStateMachineTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testEarlyAndAtWatermark() throws Exception {
  tester =
      TriggerStateMachineTester.forTrigger(
          AfterWatermarkStateMachine.pastEndOfWindow().withEarlyFirings(mockEarly),
          FixedWindows.of(Duration.millis(100)));

  injectElements(1);
  IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));

  testRunningAsTrigger(mockEarly, window);

  // Fire due to watermark
  when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false);
  tester.advanceInputWatermark(new Instant(100));
  assertTrue(tester.shouldFire(window));
  tester.fireIfShouldFire(window);
  assertTrue(tester.isMarkedFinished(window));
}
 
Example #4
Source File: BeamSqlDslJoinTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testJoinsUnboundedWithinWindowsWithDefaultTrigger() throws Exception {

  String sql =
      "SELECT o1.order_id, o1.price, o1.site_id, o2.order_id, o2.price, o2.site_id  "
          + "FROM ORDER_DETAILS1 o1"
          + " JOIN ORDER_DETAILS2 o2"
          + " on "
          + " o1.order_id=o2.site_id AND o2.price=o1.site_id";

  PCollection<Row> orders =
      ordersUnbounded()
          .apply("window", Window.into(FixedWindows.of(Duration.standardSeconds(50))));
  PCollectionTuple inputs = tuple("ORDER_DETAILS1", orders, "ORDER_DETAILS2", orders);

  PAssert.that(inputs.apply("sql", SqlTransform.query(sql)))
      .containsInAnyOrder(
          TestUtils.RowsBuilder.of(RESULT_ROW_TYPE)
              .addRows(1, 2, 2, 2, 2, 1, 1, 4, 3, 3, 3, 1)
              .getRows());

  pipeline.run();
}
 
Example #5
Source File: FlattenTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testIncompatibleWindowFnPropagationFailure() {
  p.enableAbandonedNodeEnforcement(false);

  PCollection<String> input1 =
      p.apply("CreateInput1", Create.of("Input1"))
          .apply("Window1", Window.into(FixedWindows.of(Duration.standardMinutes(1))));
  PCollection<String> input2 =
      p.apply("CreateInput2", Create.of("Input2"))
          .apply("Window2", Window.into(FixedWindows.of(Duration.standardMinutes(2))));

  try {
    PCollectionList.of(input1).and(input2).apply(Flatten.pCollections());
    Assert.fail("Exception should have been thrown");
  } catch (IllegalStateException e) {
    Assert.assertTrue(
        e.getMessage().startsWith("Inputs to Flatten had incompatible window windowFns"));
  }
}
 
Example #6
Source File: WriteFilesTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Test a WriteFiles with a windowed PCollection. */
@Test
@Category(NeedsRunner.class)
public void testWriteWindowed() throws IOException {
  List<String> inputs =
      Arrays.asList(
          "Critical canary",
          "Apprehensive eagle",
          "Intimidating pigeon",
          "Pedantic gull",
          "Frisky finch");
  runWrite(
      inputs,
      new WindowAndReshuffle<>(Window.into(FixedWindows.of(Duration.millis(2)))),
      getBaseOutputFilename(),
      WriteFiles.to(makeSimpleSink()));
}
 
Example #7
Source File: AfterFirstStateMachineTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyT1ShouldFireFixedWindows() throws Exception {
  tester =
      TriggerStateMachineTester.forTrigger(
          AfterFirstStateMachine.of(mockTrigger1, mockTrigger2),
          FixedWindows.of(Duration.millis(10)));
  tester.injectElements(1);
  IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11));

  when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true);
  when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false);

  assertTrue(tester.shouldFire(window)); // should fire

  tester.fireIfShouldFire(window);
  assertTrue(tester.isMarkedFinished(window));
}
 
Example #8
Source File: StreamWordCount.java    From beam-starter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

    Options options = PipelineOptionsFactory.fromArgs(args).withValidation()
        .as(Options.class);
    options.setRunner(FlinkRunner.class);

    Pipeline p = Pipeline.create(options);

    KafkaIO.Read<byte[], String> kafkaIOReader = KafkaIO.read()
        .withBootstrapServers("192.168.99.100:32771")
        .withTopics(Arrays.asList("beam".split(",")))
        .updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest"))
        .withValueCoder(StringUtf8Coder.of());

    p.apply(kafkaIOReader.withoutMetadata())
        .apply(Values.<String>create())
        .apply(Window.<String>into(
          FixedWindows.of(Duration.standardMinutes(options.getWindowSize()))))
        .apply(new CountWords())
        .apply(MapElements.via(new FormatAsTextFn()))
        .apply("WriteCounts", TextIO.Write.to(options.getOutput()));

    p.run();
  }
 
Example #9
Source File: LeaderBoard.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
  return infos
      .apply(
          "LeaderboardTeamFixedWindows",
          Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
              // We will get early (speculative) results as well as cumulative
              // processing of late data.
              .triggering(
                  AfterWatermark.pastEndOfWindow()
                      .withEarlyFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(FIVE_MINUTES))
                      .withLateFirings(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(TEN_MINUTES)))
              .withAllowedLateness(allowedLateness)
              .accumulatingFiredPanes())
      // Extract and sum teamname/score pairs from the event data.
      .apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
 
Example #10
Source File: PortablePipelineDotRendererTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompositePipeline() {
  p.apply(Create.timestamped(TimestampedValue.of(KV.of(1, 1), new Instant(1))))
      .apply(Window.into(FixedWindows.of(Duration.millis(10))))
      .apply(Sum.integersPerKey());

  assertEquals(
      "digraph {"
          + "    rankdir=LR"
          + "    0 [label=\"Create.TimestampedValues\\n\"]"
          + "    1 [label=\"Window.Into()\\n\"]"
          + "    0 -> 1 [style=solid label=\"Create.TimestampedValues/ParDo(ConvertTimestamps)/ParMultiDo(ConvertTimestamps).output\"]"
          + "    2 [label=\"Combine.perKey(SumInteger)\\nbeam:transform:combine_per_key:v1\"]"
          + "    1 -> 2 [style=solid label=\"Window.Into()/Window.Assign.out\"]"
          + "}",
      PortablePipelineDotRenderer.toDotString(PipelineTranslation.toProto(p))
          .replaceAll(System.lineSeparator(), ""));
}
 
Example #11
Source File: WriteFilesTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testWriteNoSpilling() throws IOException {
  List<String> inputs = Lists.newArrayList();
  for (int i = 0; i < 100; ++i) {
    inputs.add("mambo_number_" + i);
  }
  runWrite(
      inputs,
      Window.into(FixedWindows.of(Duration.millis(1))),
      getBaseOutputFilename(),
      WriteFiles.to(makeSimpleSink())
          .withMaxNumWritersPerBundle(2)
          .withWindowedWrites()
          .withNoSpilling());
}
 
Example #12
Source File: ReduceFnRunnerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * When the watermark passes the end-of-window and window expiration time in a single update, this
 * tests that it does not crash.
 */
@Test
public void testFixedWindowsEowAndGcTogether() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          FixedWindows.of(Duration.millis(10)),
          DefaultTriggerStateMachine.of(),
          AccumulationMode.ACCUMULATING_FIRED_PANES,
          Duration.millis(50),
          ClosingBehavior.FIRE_ALWAYS);

  tester.setAutoAdvanceOutputWatermark(true);

  tester.advanceInputWatermark(new Instant(0));
  injectElement(tester, 1);
  tester.advanceInputWatermark(new Instant(100));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME))));
}
 
Example #13
Source File: SumByKeyTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> counted =
      SumByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .windowBy(FixedWindows.of(org.joda.time.Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .discardingFiredPanes()
          .withAllowedLateness(Duration.millis(1000))
          .output();
  final SumByKey sum = (SumByKey) TestUtils.getProducer(counted);
  assertTrue(sum.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final WindowDesc<?> windowDesc = WindowDesc.of((Window) sum.getWindow().get());
  assertEquals(
      FixedWindows.of(org.joda.time.Duration.standardHours(1)), windowDesc.getWindowFn());
  assertEquals(DefaultTrigger.of(), windowDesc.getTrigger());
  assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, windowDesc.getAccumulationMode());
  assertEquals(Duration.millis(1000), windowDesc.getAllowedLateness());
}
 
Example #14
Source File: ReduceByKeyTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuild_sortedValues() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, List<Long>>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .reduceBy(s -> s.collect(Collectors.toList()))
          .withSortedValues(Long::compare)
          .windowBy(FixedWindows.of(Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();
  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);
  assertTrue(reduce.getValueComparator().isPresent());
}
 
Example #15
Source File: ReduceByKeyTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuild_Windowing() {
  final PCollection<String> dataset = TestUtils.createMockDataset(TypeDescriptors.strings());
  final PCollection<KV<String, Long>> reduced =
      ReduceByKey.of(dataset)
          .keyBy(s -> s)
          .valueBy(s -> 1L)
          .combineBy(Sums.ofLongs())
          .windowBy(FixedWindows.of(Duration.standardHours(1)))
          .triggeredBy(DefaultTrigger.of())
          .accumulationMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .output();

  final ReduceByKey reduce = (ReduceByKey) TestUtils.getProducer(reduced);

  assertTrue(reduce.getWindow().isPresent());
  @SuppressWarnings("unchecked")
  final Window<? extends BoundedWindow> window = (Window) reduce.getWindow().get();
  assertEquals(FixedWindows.of(org.joda.time.Duration.standardHours(1)), window.getWindowFn());
  assertEquals(DefaultTrigger.of(), WindowDesc.of(window).getTrigger());
  assertSame(
      AccumulationMode.DISCARDING_FIRED_PANES, WindowDesc.of(window).getAccumulationMode());
  assertFalse(reduce.getValueComparator().isPresent());
}
 
Example #16
Source File: ReshuffleTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterSlidingWindowsAndGroupByKey() {

  PCollection<KV<String, Iterable<Integer>>> input =
      pipeline
          .apply(
              Create.of(GBK_TESTABLE_KVS)
                  .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
          .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10L))))
          .apply(GroupByKey.create());

  PCollection<KV<String, Iterable<Integer>>> output = input.apply(Reshuffle.of());

  PAssert.that(output).satisfies(new AssertThatHasExpectedContents());

  assertEquals(input.getWindowingStrategy(), output.getWindowingStrategy());

  pipeline.run();
}
 
Example #17
Source File: DocumentationExamplesTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void windowingSection() {

  PCollection<Integer> input =
      pipeline.apply(Create.of(1, 2, 3, 4)).setTypeDescriptor(TypeDescriptors.integers());

  PCollection<KV<Integer, Long>> countedElements =
      CountByKey.of(input)
          .keyBy(e -> e)
          .windowBy(FixedWindows.of(Duration.standardSeconds(1)))
          .triggeredBy(DefaultTrigger.of())
          .discardingFiredPanes()
          .withAllowedLateness(Duration.standardSeconds(5))
          .withOnTimeBehavior(OnTimeBehavior.FIRE_IF_NON_EMPTY)
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .output();

  pipeline.run();
}
 
Example #18
Source File: GroupByKeyTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that when two elements are combined via a GroupByKey their output timestamp agrees with
 * the windowing function customized to actually be the same as the default, the earlier of the
 * two values.
 */
@Test
@Category(ValidatesRunner.class)
public void testTimestampCombinerEarliest() {

  p.apply(
          Create.timestamped(
              TimestampedValue.of(KV.of(0, "hello"), new Instant(0)),
              TimestampedValue.of(KV.of(0, "goodbye"), new Instant(10))))
      .apply(
          Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(10)))
              .withTimestampCombiner(TimestampCombiner.EARLIEST))
      .apply(GroupByKey.create())
      .apply(ParDo.of(new AssertTimestamp(new Instant(0))));

  p.run();
}
 
Example #19
Source File: ReduceFnRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testOnElementBufferingAccumulating() throws Exception {
  // Test basic execution of a trigger using a non-combining window set and accumulating mode.
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          FixedWindows.of(Duration.millis(10)),
          mockTriggerStateMachine,
          AccumulationMode.ACCUMULATING_FIRED_PANES,
          Duration.millis(100),
          ClosingBehavior.FIRE_IF_NON_EMPTY);

  injectElement(tester, 1);

  // Fires {1, 2}
  when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
  injectElement(tester, 2);

  // Fires {1, 2, 3} because we are in accumulating mode
  when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
  triggerShouldFinish(mockTriggerStateMachine);
  injectElement(tester, 3);

  // This element shouldn't be seen, because the trigger has finished
  injectElement(tester, 4);

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10),
          isSingleWindowedValue(containsInAnyOrder(1, 2, 3), 3, 0, 10)));
  assertTrue(tester.isMarkedFinished(firstWindow));
  tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
}
 
Example #20
Source File: GroupAlsoByWindowParDoFnFactoryTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testJavaWindowingStrategyDeserialization() throws Exception {
  WindowFn windowFn = FixedWindows.of(Duration.millis(17));

  WindowingStrategy windowingStrategy = WindowingStrategy.of(windowFn);

  assertThat(windowingStrategy.getWindowFn(), equalTo(windowFn));
}
 
Example #21
Source File: GroupAlsoByWindowProperties.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the given GABW implementation correctly groups elements that fall into overlapping
 * windows that are not merged.
 */
public static void groupsIntoOverlappingNonmergingWindows(
    GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

  WindowingStrategy<?, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(FixedWindows.of(Duration.millis(10)));

  List<WindowedValue<KV<String, Iterable<String>>>> result =
      runGABW(
          gabwFactory,
          windowingStrategy,
          "key",
          WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING),
          WindowedValue.of("v2", new Instant(4), Arrays.asList(window(1, 5)), PaneInfo.NO_FIRING),
          WindowedValue.of(
              "v3", new Instant(4), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING));

  assertThat(result, hasSize(2));

  TimestampedValue<KV<String, Iterable<String>>> item0 =
      getOnlyElementInWindow(result, window(0, 5));
  assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v3"));
  assertThat(item0.getTimestamp(), equalTo(window(1, 5).maxTimestamp()));

  TimestampedValue<KV<String, Iterable<String>>> item1 =
      getOnlyElementInWindow(result, window(1, 5));
  assertThat(item1.getValue().getValue(), contains("v2"));
  assertThat(item1.getTimestamp(), equalTo(window(0, 5).maxTimestamp()));
}
 
Example #22
Source File: BeamModel.java    From streamingbook with Apache License 2.0 5 votes vote down vote up
@Override
public PCollection<String> expand(PCollection<KV<String, Integer>> input) {
    return input
        .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)))
        .apply(Sum.integersPerKey())
        .apply(ParDo.of(new FormatAsStrings()));
}
 
Example #23
Source File: ParDoTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
@Category(ValidatesRunner.class)
public void testWindowingInStartAndFinishBundle() {

  final FixedWindows windowFn = FixedWindows.of(Duration.millis(1));
  PCollection<String> output =
      pipeline
          .apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1))))
          .apply(Window.into(windowFn))
          .apply(
              ParDo.of(
                  new DoFn<String, String>() {
                    @ProcessElement
                    public void processElement(
                        @Element String element,
                        @Timestamp Instant timestamp,
                        OutputReceiver<String> r) {
                      r.output(element);
                      System.out.println("Process: " + element + ":" + timestamp.getMillis());
                    }

                    @FinishBundle
                    public void finishBundle(FinishBundleContext c) {
                      Instant ts = new Instant(3);
                      c.output("finish", ts, windowFn.assignWindow(ts));
                      System.out.println("Finish: 3");
                    }
                  }))
          .apply(ParDo.of(new PrintingDoFn()));

  PAssert.that(output).satisfies(new Checker());

  pipeline.run();
}
 
Example #24
Source File: DistinctTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleDuplicatesWithStreamStrategyOldest() {
  execute(
      new AbstractTestCase<KV<String, Long>, String>() {

        @Override
        public List<String> getUnorderedOutput() {
          return Arrays.asList("2", "1", "3");
        }

        @Override
        protected PCollection<String> getOutput(PCollection<KV<String, Long>> input) {
          input = AssignEventTime.of(input).using(KV::getValue).output();
          PCollection<KV<String, Long>> distinct =
              Distinct.of(input)
                  .projected(
                      in -> in.getKey().substring(0, 1),
                      Distinct.SelectionPolicy.OLDEST,
                      TypeDescriptors.strings())
                  .windowBy(FixedWindows.of(org.joda.time.Duration.standardSeconds(1)))
                  .triggeredBy(DefaultTrigger.of())
                  .discardingFiredPanes()
                  .output();
          return MapElements.of(distinct).using(KV::getKey).output();
        }

        @Override
        protected List<KV<String, Long>> getInput() {
          return asTimedList(100, "1", "2", "3", "3.", "2.", "1.");
        }

        @Override
        protected TypeDescriptor<KV<String, Long>> getInputType() {
          return TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs());
        }
      });
}
 
Example #25
Source File: TopTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testTopEmptyWithIncompatibleWindows() {
  p.enableAbandonedNodeEnforcement(false);

  Window<String> windowingFn = Window.into(FixedWindows.of(Duration.standardDays(10L)));
  PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())).apply(windowingFn);

  expectedEx.expect(IllegalStateException.class);
  expectedEx.expectMessage("Top");
  expectedEx.expectMessage("GlobalWindows");
  expectedEx.expectMessage("withoutDefaults");
  expectedEx.expectMessage("asSingletonView");

  input.apply(Top.of(1, new OrderByLength()));
}
 
Example #26
Source File: ReduceFnRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/** Tests that a processing time timer does not cause window GC. */
@Test
public void testProcessingTimeTimerDoesNotGc() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withTrigger(
              Repeatedly.forever(
                  AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceProcessingTime(new Instant(5000));
  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceProcessingTime(new Instant(10000));

  tester.assertHasOnlyGlobalAndStateFor(new IntervalWindow(new Instant(0), new Instant(100)));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0))));
}
 
Example #27
Source File: WindowIntoTranslationTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Parameters(name = "{index}: {0}")
public static Iterable<WindowFn<?, ?>> data() {
  // This pipeline exists for construction, not to run any test.
  return ImmutableList.<WindowFn<?, ?>>builder()
      .add(FixedWindows.of(Duration.standardMinutes(10L)))
      .add(new GlobalWindows())
      .add(Sessions.withGapDuration(Duration.standardMinutes(15L)))
      .add(SlidingWindows.of(Duration.standardMinutes(5L)).every(Duration.standardMinutes(1L)))
      .add(new CustomWindows())
      .build();
}
 
Example #28
Source File: LateDataUtilsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void beforeEndOfGlobalWindowSame() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.standardMinutes(2);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(10));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(window.maxTimestamp().plus(allowedLateness)));
}
 
Example #29
Source File: RepeatedlyStateMachineTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/** Tests that onElement correctly passes the data on to the subtrigger. */
@Test
public void testOnElement() throws Exception {
  setUp(FixedWindows.of(Duration.millis(10)));
  tester.injectElements(37);
  verify(mockTrigger).onElement(Mockito.any());
}
 
Example #30
Source File: ReduceFnRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void noEmptyPanesFinalAlways() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
              .withTrigger(
                  Repeatedly.forever(
                      AfterFirst.of(
                          AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow())))
              .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
              .withAllowedLateness(Duration.millis(100))
              .withTimestampCombiner(TimestampCombiner.EARLIEST)
              .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));

  tester.advanceInputWatermark(new Instant(0));
  tester.injectElements(
      TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
  tester.advanceInputWatermark(new Instant(20));
  tester.advanceInputWatermark(new Instant(250));

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertThat(
      output,
      contains(
          // Trigger with 2 elements
          isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10),
          // Trigger for the empty on time pane
          isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10),
          // Trigger for the final pane
          isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}