Java Code Examples for org.apache.beam.sdk.transforms.windowing.GlobalWindow

The following examples show how to use org.apache.beam.sdk.transforms.windowing.GlobalWindow. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: beam   Source File: PaneExtractorsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void nonLatePanesSingleEarly() {
  SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
      PaneExtractors.nonLatePanes();
  Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
      ImmutableList.of(
          ValueInSingleWindow.of(
              8,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(true, false, Timing.EARLY)),
          ValueInSingleWindow.of(
              4,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(true, false, Timing.EARLY)));

  assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8));
}
 
Example 2
Source Project: deployment-examples   Source File: StatefulTeamScoreTest.java    License: MIT License 6 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for one team.
 */
@Test
public void testScoreUpdatesOneTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)),
              event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)),
              event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)),
              event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)),
              event(TestUser.RED_TWO, 201, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401));

  p.run().waitUntilFinish();
}
 
Example 3
Source Project: beam   Source File: LengthPrefixUnknownCodersTest.java    License: Apache License 2.0 6 votes vote down vote up
/** Test bypassing unknown coders that are already wrapped with {@code LengthPrefixCoder} */
@Test
public void testLengthPrefixForLengthPrefixCoder() throws Exception {
  Coder<WindowedValue<KV<String, Integer>>> windowedValueCoder =
      WindowedValue.getFullCoder(
          KvCoder.of(StringUtf8Coder.of(), LengthPrefixCoder.of(VarIntCoder.of())),
          GlobalWindow.Coder.INSTANCE);

  Map<String, Object> lengthPrefixedCoderCloudObject =
      forCodec(CloudObjects.asCloudObject(windowedValueCoder, /*sdkComponents=*/ null), false);

  Coder<WindowedValue<KV<String, Integer>>> expectedCoder =
      WindowedValue.getFullCoder(
          KvCoder.of(StringUtf8Coder.of(), LengthPrefixCoder.of(VarIntCoder.of())),
          GlobalWindow.Coder.INSTANCE);

  assertEqualsAsJson(
      CloudObjects.asCloudObject(expectedCoder, /*sdkComponents=*/ null),
      lengthPrefixedCoderCloudObject);
}
 
Example 4
Source Project: beam   Source File: StatefulTeamScoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for one team.
 */
@Test
public void testScoreUpdatesOneTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)),
              event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)),
              event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)),
              event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)),
              event(TestUser.RED_TWO, 201, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401));

  p.run().waitUntilFinish();
}
 
Example 5
@Test
public void processElementSideInputNotReadyMultipleWindows() {
  when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class)))
      .thenReturn(false);

  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner =
      createRunner(ImmutableList.of(singletonView));

  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(
              new IntervalWindow(new Instant(-500L), new Instant(0L)),
              new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)),
              GlobalWindow.INSTANCE),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(multiWindowPushback, equalTo(multiWindow.explodeWindows()));
  assertThat(underlying.inputElems, emptyIterable());
}
 
Example 6
Source Project: beam   Source File: StateNamespacesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test should not be changed. It verifies that the stringKey matches certain expectations.
 * If this changes, the ability to reload any pipeline that has persisted these namespaces will be
 * impacted.
 */
@Test
public void testStability() {
  StateNamespace global = StateNamespaces.global();
  StateNamespace intervalWindow =
      StateNamespaces.window(intervalCoder, intervalWindow(1000, 87392));
  StateNamespace intervalWindowAndTrigger =
      StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 87392), 57);
  StateNamespace globalWindow =
      StateNamespaces.window(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE);
  StateNamespace globalWindowAndTrigger =
      StateNamespaces.windowAndTrigger(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, 12);

  assertEquals("/", global.stringKey());
  assertEquals("/gAAAAAABVWD4ogU/", intervalWindow.stringKey());
  assertEquals("/gAAAAAABVWD4ogU/1L/", intervalWindowAndTrigger.stringKey());
  assertEquals("//", globalWindow.stringKey());
  assertEquals("//C/", globalWindowAndTrigger.stringKey());
}
 
Example 7
Source Project: beam   Source File: BigtableIOTest.java    License: Apache License 2.0 6 votes vote down vote up
/** Tests that at least one result is emitted per element written in the global window. */
@Test
public void testWritingEmitsResultsWhenDoneInGlobalWindow() throws Exception {
  final String table = "table";
  final String key = "key";
  final String value = "value";

  service.createTable(table);

  PCollection<BigtableWriteResult> results =
      p.apply("single row", Create.of(makeWrite(key, value)).withCoder(bigtableCoder))
          .apply("write", defaultWrite.withTableId(table).withWriteResults());
  PAssert.that(results)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(BigtableWriteResult.create(1));

  p.run();
}
 
Example 8
Source Project: beam   Source File: BatchViewOverridesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBatchViewAsSingletonToIsmRecord() throws Exception {
  DoFnTester<
          KV<Integer, Iterable<KV<GlobalWindow, WindowedValue<String>>>>,
          IsmRecord<WindowedValue<String>>>
      doFnTester =
          DoFnTester.of(
              new BatchViewOverrides.BatchViewAsSingleton.IsmRecordForSingularValuePerWindowDoFn<
                  String, GlobalWindow>(GlobalWindow.Coder.INSTANCE));

  assertThat(
      doFnTester.processBundle(
          ImmutableList.of(
              KV.of(
                  0, ImmutableList.of(KV.of(GlobalWindow.INSTANCE, valueInGlobalWindow("a")))))),
      contains(IsmRecord.of(ImmutableList.of(GlobalWindow.INSTANCE), valueInGlobalWindow("a"))));
}
 
Example 9
Source Project: beam   Source File: WindowMappingFnRunnerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWindowMapping() throws Exception {
  String pTransformId = "pTransformId";

  SdkComponents components = SdkComponents.create();
  components.registerEnvironment(Environments.createDockerEnvironment("java"));
  RunnerApi.FunctionSpec functionSpec =
      RunnerApi.FunctionSpec.newBuilder()
          .setUrn(WindowMappingFnRunner.URN)
          .setPayload(
              ParDoTranslation.translateWindowMappingFn(
                      new GlobalWindows().getDefaultWindowMappingFn(), components)
                  .toByteString())
          .build();
  RunnerApi.PTransform pTransform =
      RunnerApi.PTransform.newBuilder().setSpec(functionSpec).build();

  ThrowingFunction<KV<Object, BoundedWindow>, KV<Object, BoundedWindow>> mapFunction =
      WindowMappingFnRunner.createMapFunctionForPTransform(pTransformId, pTransform);

  KV<Object, BoundedWindow> input =
      KV.of("abc", new IntervalWindow(Instant.now(), Duration.standardMinutes(1)));

  assertEquals(KV.of(input.getKey(), GlobalWindow.INSTANCE), mapFunction.apply(input));
}
 
Example 10
Source Project: beam   Source File: RowHelpers.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Serialize a windowedValue to bytes using windowedValueCoder {@link
 * WindowedValue.FullWindowedValueCoder} and stores it an InternalRow.
 */
public static <T> InternalRow storeWindowedValueInRow(
    WindowedValue<T> windowedValue, Coder<T> coder) {
  List<Object> list = new ArrayList<>();
  // serialize the windowedValue to bytes array to comply with dataset binary schema
  WindowedValue.FullWindowedValueCoder<T> windowedValueCoder =
      WindowedValue.FullWindowedValueCoder.of(coder, GlobalWindow.Coder.INSTANCE);
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try {
    windowedValueCoder.encode(windowedValue, byteArrayOutputStream);
    byte[] bytes = byteArrayOutputStream.toByteArray();
    list.add(bytes);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  return InternalRow.apply(asScalaBuffer(list).toList());
}
 
Example 11
Source Project: beam   Source File: SideInputContainerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void isReadyForEmptyWindowTrue() throws Exception {
  CountDownLatch onComplete = new CountDownLatch(1);
  immediatelyInvokeCallback(mapView, GlobalWindow.INSTANCE);
  CountDownLatch latch = invokeLatchedCallback(singletonView, GlobalWindow.INSTANCE, onComplete);

  ReadyCheckingSideInputReader reader =
      container.createReaderForViews(ImmutableList.of(mapView, singletonView));
  assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(true));
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));

  latch.countDown();
  if (!onComplete.await(1500L, TimeUnit.MILLISECONDS)) {
    fail("Callback to set empty values did not complete!");
  }
  // The cached value was false, so it continues to be true
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));

  // A new reader for the same container gets a fresh look
  reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(true));
}
 
Example 12
Source Project: beam   Source File: WriteFiles.java    License: Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void process(ProcessContext c) throws Exception {
  getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
  @Nullable Integer fixedNumShards;
  if (numShardsView != null) {
    fixedNumShards = c.sideInput(numShardsView);
  } else if (getNumShardsProvider() != null) {
    fixedNumShards = getNumShardsProvider().get();
  } else {
    fixedNumShards = null;
  }
  List<FileResult<DestinationT>> fileResults = Lists.newArrayList(c.element());
  LOG.info("Finalizing {} file results", fileResults.size());
  DestinationT defaultDest = getDynamicDestinations().getDefaultDestination();
  List<KV<FileResult<DestinationT>, ResourceId>> resultsToFinalFilenames =
      fileResults.isEmpty()
          ? writeOperation.finalizeDestination(
              defaultDest, GlobalWindow.INSTANCE, fixedNumShards, fileResults)
          : finalizeAllDestinations(fileResults, fixedNumShards);
  writeOperation.moveToOutputFiles(resultsToFinalFilenames);
  for (KV<FileResult<DestinationT>, ResourceId> entry : resultsToFinalFilenames) {
    FileResult<DestinationT> res = entry.getKey();
    c.output(KV.of(res.getDestination(), entry.getValue().toString()));
  }
}
 
Example 13
Source Project: beam   Source File: SdkHarnessClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRegister() throws Exception {
  ProcessBundleDescriptor descriptor1 =
      ProcessBundleDescriptor.newBuilder().setId("descriptor1").build();

  List<RemoteInputDestination> remoteInputs =
      Collections.singletonList(
          RemoteInputDestination.of(
              (FullWindowedValueCoder)
                  FullWindowedValueCoder.of(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE),
              SDK_GRPC_READ_TRANSFORM));

  BundleProcessor processor1 = sdkHarnessClient.getProcessor(descriptor1, remoteInputs);

  verify(fnApiControlClient).registerProcessBundleDescriptor(descriptor1);
}
 
Example 14
Source Project: beam   Source File: FakeDatasetService.java    License: Apache License 2.0 6 votes vote down vote up
public long insertAll(
    TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
    throws IOException, InterruptedException {
  List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList();
  for (TableRow row : rowList) {
    windowedRows.add(
        ValueInSingleWindow.of(
            row,
            GlobalWindow.TIMESTAMP_MAX_VALUE,
            GlobalWindow.INSTANCE,
            PaneInfo.ON_TIME_AND_ONLY_FIRING));
  }
  return insertAll(
      ref,
      windowedRows,
      insertIdList,
      InsertRetryPolicy.alwaysRetry(),
      null,
      null,
      false,
      false,
      false);
}
 
Example 15
Source Project: beam   Source File: ProcessFnRunner.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> void checkTrivialOuterWindows(
    WindowedValue<KeyedWorkItem<byte[], T>> windowedKWI) {
  // In practice it will be in 0 or 1 windows (ValueInEmptyWindows or ValueInGlobalWindow)
  Collection<? extends BoundedWindow> outerWindows = windowedKWI.getWindows();
  if (!outerWindows.isEmpty()) {
    checkArgument(
        outerWindows.size() == 1,
        "The KeyedWorkItem itself must not be in multiple windows, but was in: %s",
        outerWindows);
    BoundedWindow onlyWindow = Iterables.getOnlyElement(outerWindows);
    checkArgument(
        onlyWindow instanceof GlobalWindow,
        "KeyedWorkItem must be in the Global window, but was in: %s",
        onlyWindow);
  }
}
 
Example 16
Source Project: beam   Source File: SdkHarnessClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRegisterWithStateRequiresStateDelegator() throws Exception {
  ProcessBundleDescriptor descriptor =
      ProcessBundleDescriptor.newBuilder()
          .setId("test")
          .setStateApiServiceDescriptor(ApiServiceDescriptor.newBuilder().setUrl("foo"))
          .build();

  List<RemoteInputDestination> remoteInputs =
      Collections.singletonList(
          RemoteInputDestination.of(
              (FullWindowedValueCoder)
                  FullWindowedValueCoder.of(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE),
              SDK_GRPC_READ_TRANSFORM));

  thrown.expect(IllegalStateException.class);
  thrown.expectMessage("containing a state");
  sdkHarnessClient.getProcessor(descriptor, remoteInputs);
}
 
Example 17
Source Project: beam   Source File: LateDataUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(-100));
  assertThat(
      window.maxTimestamp().plus(allowedLateness),
      Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 
Example 18
Source Project: streamingbook   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static String formatTime(Instant timestamp) {
if (timestamp == null) {
    return "null";
} else if (timestamp.equals(BoundedWindow.TIMESTAMP_MIN_VALUE)) {
           return "TIMESTAMP_MIN_VALUE";
       } else if (timestamp.equals(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
           return "TIMESTAMP_MAX_VALUE";
       } else if (timestamp.equals(GlobalWindow.INSTANCE.maxTimestamp())) {
           return "END_OF_GLOBAL_WINDOW";
       } else {
           return timestamp.toString(TIME_FMT);
       }
   }
 
Example 19
Source Project: incubator-nemo   Source File: BeamUnboundedSourceVertex.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long readWatermark() {
  final Instant watermark = reader.getWatermark();
  // Finish if the watermark == TIMESTAMP_MAX_VALUE
  isFinished = (watermark.getMillis() >= GlobalWindow.TIMESTAMP_MAX_VALUE.getMillis());
  return watermark.getMillis();
}
 
Example 20
Source Project: feast   Source File: WriteFeatureSetSpecAckTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void shouldSendAckWhenAllSinksReady() {
  TestStream<FeatureSetReference> sink1 =
      TestStream.create(AvroCoder.of(FeatureSetReference.class))
          .addElements(FeatureSetReference.of("project", "fs", 1))
          .addElements(FeatureSetReference.of("project", "fs", 2))
          .addElements(FeatureSetReference.of("project", "fs", 3))
          .advanceWatermarkToInfinity();

  TestStream<FeatureSetReference> sink2 =
      TestStream.create(AvroCoder.of(FeatureSetReference.class))
          .addElements(FeatureSetReference.of("project", "fs_2", 1))
          .addElements(FeatureSetReference.of("project", "fs", 3))
          .advanceWatermarkToInfinity();

  TestStream<FeatureSetReference> sink3 =
      TestStream.create(AvroCoder.of(FeatureSetReference.class))
          .advanceProcessingTime(Duration.standardSeconds(10))
          .addElements(FeatureSetReference.of("project", "fs", 3))
          .advanceWatermarkToInfinity();

  PCollectionList<FeatureSetReference> sinks =
      PCollectionList.of(
          ImmutableList.of(
              p.apply("sink1", sink1), p.apply("sink2", sink2), p.apply("sink3", sink3)));

  PCollection<FeatureSetReference> grouped =
      sinks.apply(Flatten.pCollections()).apply(new WriteFeatureSetSpecAck.PrepareWrite(3));

  PAssert.that(grouped)
      .inOnTimePane(GlobalWindow.INSTANCE)
      .containsInAnyOrder(FeatureSetReference.of("project", "fs", 3));

  p.run();
}
 
Example 21
Source Project: deployment-examples   Source File: StatefulTeamScoreTest.java    License: MIT License 5 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for multiple teams.
 */
@Test
public void testScoreUpdatesPerTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_ONE, 50, Duration.standardSeconds(10)),
              event(TestUser.RED_TWO, 50, Duration.standardSeconds(20)),
              event(TestUser.BLUE_ONE, 70, Duration.standardSeconds(30)),
              event(TestUser.BLUE_TWO, 80, Duration.standardSeconds(40)),
              event(TestUser.BLUE_TWO, 50, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();
  String blueTeam = TestUser.BLUE_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(blueTeam, 150), KV.of(blueTeam, 200));

  p.run().waitUntilFinish();
}
 
Example 22
Source Project: beam   Source File: BigQueryUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInsertAll() throws Exception {
  // Build up a list of indices to fail on each invocation. This should result in
  // 5 calls to insertAll.
  List<List<Long>> errorsIndices = new ArrayList<>();
  errorsIndices.add(Arrays.asList(0L, 5L, 10L, 15L, 20L));
  errorsIndices.add(Arrays.asList(0L, 2L, 4L));
  errorsIndices.add(Arrays.asList(0L, 2L));
  errorsIndices.add(new ArrayList<>());
  onInsertAll(errorsIndices);

  TableReference ref = BigQueryHelpers.parseTableSpec("project:dataset.table");
  DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, options, 5);

  List<ValueInSingleWindow<TableRow>> rows = new ArrayList<>();
  List<String> ids = new ArrayList<>();
  for (int i = 0; i < 25; ++i) {
    rows.add(
        ValueInSingleWindow.of(
            rawRow("foo", 1234),
            GlobalWindow.TIMESTAMP_MAX_VALUE,
            GlobalWindow.INSTANCE,
            PaneInfo.ON_TIME_AND_ONLY_FIRING));
    ids.add("");
  }

  long totalBytes = 0;
  try {
    totalBytes =
        datasetService.insertAll(
            ref, rows, ids, InsertRetryPolicy.alwaysRetry(), null, null, false, false, false);
  } finally {
    verifyInsertAll(5);
    // Each of the 25 rows has 1 byte for length and 30 bytes: '{"f":[{"v":"foo"},{"v":1234}]}'
    assertEquals("Incorrect byte count", 25L * 31L, totalBytes);
  }
}
 
Example 23
Source Project: beam   Source File: LateDataUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindow() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault().withWindowFn(windowFn);

  IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
  assertThat(window.maxTimestamp(), equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 
Example 24
Source Project: beam   Source File: StatefulTeamScoreTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for multiple teams.
 */
@Test
public void testScoreUpdatesPerTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_ONE, 50, Duration.standardSeconds(10)),
              event(TestUser.RED_TWO, 50, Duration.standardSeconds(20)),
              event(TestUser.BLUE_ONE, 70, Duration.standardSeconds(30)),
              event(TestUser.BLUE_TWO, 80, Duration.standardSeconds(40)),
              event(TestUser.BLUE_TWO, 50, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();
  String blueTeam = TestUser.BLUE_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(blueTeam, 150), KV.of(blueTeam, 200));

  p.run().waitUntilFinish();
}
 
Example 25
Source Project: beam   Source File: SourceInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public WindowedValue<T> nextRecord(WindowedValue<T> t) throws IOException {
  if (inputAvailable) {
    final T current = reader.getCurrent();
    final Instant timestamp = reader.getCurrentTimestamp();
    // advance reader to have a record ready next time
    inputAvailable = readerInvoker.invokeAdvance(reader);
    return WindowedValue.of(current, timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
  }

  return null;
}
 
Example 26
Source Project: beam   Source File: KeyedPValueTrackingVisitorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void keyedInputWithKeyPreserving() {

  PCollection<KV<String, WindowedValue<KV<String, Integer>>>> input =
      p.apply(
          Create.of(
                  KV.of(
                      "hello",
                      WindowedValue.of(
                          KV.of("hello", 3),
                          new Instant(0),
                          new IntervalWindow(new Instant(0), new Instant(9)),
                          PaneInfo.NO_FIRING)))
              .withCoder(
                  KvCoder.of(
                      StringUtf8Coder.of(),
                      WindowedValue.getValueOnlyCoder(
                          KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))));

  TupleTag<KeyedWorkItem<String, KV<String, Integer>>> keyedTag = new TupleTag<>();
  PCollection<KeyedWorkItem<String, KV<String, Integer>>> keyed =
      input
          .apply(new DirectGroupByKeyOnly<>())
          .apply(
              new DirectGroupAlsoByWindow<>(
                  WindowingStrategy.globalDefault(), WindowingStrategy.globalDefault()))
          .apply(
              ParDo.of(new ParDoMultiOverrideFactory.ToKeyedWorkItem<String, Integer>())
                  .withOutputTags(keyedTag, TupleTagList.empty()))
          .get(keyedTag)
          .setCoder(
              KeyedWorkItemCoder.of(
                  StringUtf8Coder.of(),
                  KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()),
                  GlobalWindow.Coder.INSTANCE));

  p.traverseTopologically(visitor);
  assertThat(visitor.getKeyedPValues(), hasItem(keyed));
}
 
Example 27
Source Project: beam   Source File: BatchViewOverrides.java    License: Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
  c.output(
      IsmRecord.of(
          ImmutableList.of(GlobalWindow.INSTANCE, indexInBundle),
          WindowedValue.of(c.element(), c.timestamp(), GlobalWindow.INSTANCE, c.pane())));
  indexInBundle += 1;
}
 
Example 28
Source Project: beam   Source File: ReduceFnRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to
 * end-of-time.
 */
@Test
public void fireNonEmptyOnDrainInGlobalWindow() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(new GlobalWindows())
              .withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3)))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES));

  tester.advanceInputWatermark(new Instant(0));

  final int n = 20;
  for (int i = 0; i < n; i++) {
    tester.injectElements(TimestampedValue.of(i, new Instant(i)));
  }

  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertEquals(n / 3, output.size());
  for (int i = 0; i < output.size(); i++) {
    assertEquals(Timing.EARLY, output.get(i).getPane().getTiming());
    assertEquals(i, output.get(i).getPane().getIndex());
    assertEquals(3, Iterables.size(output.get(i).getValue()));
  }

  tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);

  output = tester.extractOutput();
  assertEquals(1, output.size());
  assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming());
  assertEquals(n / 3, output.get(0).getPane().getIndex());
  assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue()));
}
 
Example 29
Source Project: beam   Source File: EvaluationContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void callAfterOutputMustHaveBeenProducedAlreadyAfterCallsImmediately() throws Exception {
  TransformResult<?> finishedResult = StepTransformResult.withoutHold(createdProducer).build();
  context.handleResult(null, ImmutableList.of(), finishedResult);

  final CountDownLatch callLatch = new CountDownLatch(1);
  context.extractFiredTimers();
  Runnable callback = callLatch::countDown;
  context.scheduleAfterOutputWouldBeProduced(
      downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback);
  assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true));
}
 
Example 30
Source Project: beam   Source File: FnApiDoFnRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Produces an iterable side input {@link StateKey} for the test PTransform id in the global
 * window.
 */
private StateKey iterableSideInputKey(String sideInputId) throws IOException {
  return iterableSideInputKey(
      sideInputId,
      ByteString.copyFrom(
          CoderUtils.encodeToByteArray(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE)));
}