org.apache.beam.sdk.transforms.windowing.GlobalWindow Java Examples

The following examples show how to use org.apache.beam.sdk.transforms.windowing.GlobalWindow. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SideInputContainerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void isReadyForEmptyWindowTrue() throws Exception {
  CountDownLatch onComplete = new CountDownLatch(1);
  immediatelyInvokeCallback(mapView, GlobalWindow.INSTANCE);
  CountDownLatch latch = invokeLatchedCallback(singletonView, GlobalWindow.INSTANCE, onComplete);

  ReadyCheckingSideInputReader reader =
      container.createReaderForViews(ImmutableList.of(mapView, singletonView));
  assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(true));
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));

  latch.countDown();
  if (!onComplete.await(1500L, TimeUnit.MILLISECONDS)) {
    fail("Callback to set empty values did not complete!");
  }
  // The cached value was false, so it continues to be true
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));

  // A new reader for the same container gets a fresh look
  reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
  assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(true));
}
 
Example #2
Source File: WindowMappingFnRunnerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testWindowMapping() throws Exception {
  String pTransformId = "pTransformId";

  SdkComponents components = SdkComponents.create();
  components.registerEnvironment(Environments.createDockerEnvironment("java"));
  RunnerApi.FunctionSpec functionSpec =
      RunnerApi.FunctionSpec.newBuilder()
          .setUrn(WindowMappingFnRunner.URN)
          .setPayload(
              ParDoTranslation.translateWindowMappingFn(
                      new GlobalWindows().getDefaultWindowMappingFn(), components)
                  .toByteString())
          .build();
  RunnerApi.PTransform pTransform =
      RunnerApi.PTransform.newBuilder().setSpec(functionSpec).build();

  ThrowingFunction<KV<Object, BoundedWindow>, KV<Object, BoundedWindow>> mapFunction =
      WindowMappingFnRunner.createMapFunctionForPTransform(pTransformId, pTransform);

  KV<Object, BoundedWindow> input =
      KV.of("abc", new IntervalWindow(Instant.now(), Duration.standardMinutes(1)));

  assertEquals(KV.of(input.getKey(), GlobalWindow.INSTANCE), mapFunction.apply(input));
}
 
Example #3
Source File: SdkHarnessClientTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testRegister() throws Exception {
  ProcessBundleDescriptor descriptor1 =
      ProcessBundleDescriptor.newBuilder().setId("descriptor1").build();

  List<RemoteInputDestination> remoteInputs =
      Collections.singletonList(
          RemoteInputDestination.of(
              (FullWindowedValueCoder)
                  FullWindowedValueCoder.of(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE),
              SDK_GRPC_READ_TRANSFORM));

  BundleProcessor processor1 = sdkHarnessClient.getProcessor(descriptor1, remoteInputs);

  verify(fnApiControlClient).registerProcessBundleDescriptor(descriptor1);
}
 
Example #4
Source File: BatchViewOverridesTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testBatchViewAsSingletonToIsmRecord() throws Exception {
  DoFnTester<
          KV<Integer, Iterable<KV<GlobalWindow, WindowedValue<String>>>>,
          IsmRecord<WindowedValue<String>>>
      doFnTester =
          DoFnTester.of(
              new BatchViewOverrides.BatchViewAsSingleton.IsmRecordForSingularValuePerWindowDoFn<
                  String, GlobalWindow>(GlobalWindow.Coder.INSTANCE));

  assertThat(
      doFnTester.processBundle(
          ImmutableList.of(
              KV.of(
                  0, ImmutableList.of(KV.of(GlobalWindow.INSTANCE, valueInGlobalWindow("a")))))),
      contains(IsmRecord.of(ImmutableList.of(GlobalWindow.INSTANCE), valueInGlobalWindow("a"))));
}
 
Example #5
Source File: BigtableIOTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that at least one result is emitted per element written in the global window. */
@Test
public void testWritingEmitsResultsWhenDoneInGlobalWindow() throws Exception {
  final String table = "table";
  final String key = "key";
  final String value = "value";

  service.createTable(table);

  PCollection<BigtableWriteResult> results =
      p.apply("single row", Create.of(makeWrite(key, value)).withCoder(bigtableCoder))
          .apply("write", defaultWrite.withTableId(table).withWriteResults());
  PAssert.that(results)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(BigtableWriteResult.create(1));

  p.run();
}
 
Example #6
Source File: StateNamespacesTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * This test should not be changed. It verifies that the stringKey matches certain expectations.
 * If this changes, the ability to reload any pipeline that has persisted these namespaces will be
 * impacted.
 */
@Test
public void testStability() {
  StateNamespace global = StateNamespaces.global();
  StateNamespace intervalWindow =
      StateNamespaces.window(intervalCoder, intervalWindow(1000, 87392));
  StateNamespace intervalWindowAndTrigger =
      StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 87392), 57);
  StateNamespace globalWindow =
      StateNamespaces.window(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE);
  StateNamespace globalWindowAndTrigger =
      StateNamespaces.windowAndTrigger(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, 12);

  assertEquals("/", global.stringKey());
  assertEquals("/gAAAAAABVWD4ogU/", intervalWindow.stringKey());
  assertEquals("/gAAAAAABVWD4ogU/1L/", intervalWindowAndTrigger.stringKey());
  assertEquals("//", globalWindow.stringKey());
  assertEquals("//C/", globalWindowAndTrigger.stringKey());
}
 
Example #7
Source File: FakeDatasetService.java    From beam with Apache License 2.0 6 votes vote down vote up
public long insertAll(
    TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList)
    throws IOException, InterruptedException {
  List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList();
  for (TableRow row : rowList) {
    windowedRows.add(
        ValueInSingleWindow.of(
            row,
            GlobalWindow.TIMESTAMP_MAX_VALUE,
            GlobalWindow.INSTANCE,
            PaneInfo.ON_TIME_AND_ONLY_FIRING));
  }
  return insertAll(
      ref,
      windowedRows,
      insertIdList,
      InsertRetryPolicy.alwaysRetry(),
      null,
      null,
      false,
      false,
      false);
}
 
Example #8
Source File: SimplePushbackSideInputDoFnRunnerTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void processElementSideInputNotReadyMultipleWindows() {
  when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class)))
      .thenReturn(false);

  SimplePushbackSideInputDoFnRunner<Integer, Integer> runner =
      createRunner(ImmutableList.of(singletonView));

  WindowedValue<Integer> multiWindow =
      WindowedValue.of(
          2,
          new Instant(-2),
          ImmutableList.of(
              new IntervalWindow(new Instant(-500L), new Instant(0L)),
              new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)),
              GlobalWindow.INSTANCE),
          PaneInfo.ON_TIME_AND_ONLY_FIRING);
  Iterable<WindowedValue<Integer>> multiWindowPushback =
      runner.processElementInReadyWindows(multiWindow);
  assertThat(multiWindowPushback, equalTo(multiWindow.explodeWindows()));
  assertThat(underlying.inputElems, emptyIterable());
}
 
Example #9
Source File: StatefulTeamScoreTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for one team.
 */
@Test
public void testScoreUpdatesOneTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)),
              event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)),
              event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)),
              event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)),
              event(TestUser.RED_TWO, 201, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401));

  p.run().waitUntilFinish();
}
 
Example #10
Source File: ProcessFnRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <T> void checkTrivialOuterWindows(
    WindowedValue<KeyedWorkItem<byte[], T>> windowedKWI) {
  // In practice it will be in 0 or 1 windows (ValueInEmptyWindows or ValueInGlobalWindow)
  Collection<? extends BoundedWindow> outerWindows = windowedKWI.getWindows();
  if (!outerWindows.isEmpty()) {
    checkArgument(
        outerWindows.size() == 1,
        "The KeyedWorkItem itself must not be in multiple windows, but was in: %s",
        outerWindows);
    BoundedWindow onlyWindow = Iterables.getOnlyElement(outerWindows);
    checkArgument(
        onlyWindow instanceof GlobalWindow,
        "KeyedWorkItem must be in the Global window, but was in: %s",
        onlyWindow);
  }
}
 
Example #11
Source File: SdkHarnessClientTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testRegisterWithStateRequiresStateDelegator() throws Exception {
  ProcessBundleDescriptor descriptor =
      ProcessBundleDescriptor.newBuilder()
          .setId("test")
          .setStateApiServiceDescriptor(ApiServiceDescriptor.newBuilder().setUrl("foo"))
          .build();

  List<RemoteInputDestination> remoteInputs =
      Collections.singletonList(
          RemoteInputDestination.of(
              (FullWindowedValueCoder)
                  FullWindowedValueCoder.of(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE),
              SDK_GRPC_READ_TRANSFORM));

  thrown.expect(IllegalStateException.class);
  thrown.expectMessage("containing a state");
  sdkHarnessClient.getProcessor(descriptor, remoteInputs);
}
 
Example #12
Source File: RowHelpers.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize a windowedValue to bytes using windowedValueCoder {@link
 * WindowedValue.FullWindowedValueCoder} and stores it an InternalRow.
 */
public static <T> InternalRow storeWindowedValueInRow(
    WindowedValue<T> windowedValue, Coder<T> coder) {
  List<Object> list = new ArrayList<>();
  // serialize the windowedValue to bytes array to comply with dataset binary schema
  WindowedValue.FullWindowedValueCoder<T> windowedValueCoder =
      WindowedValue.FullWindowedValueCoder.of(coder, GlobalWindow.Coder.INSTANCE);
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try {
    windowedValueCoder.encode(windowedValue, byteArrayOutputStream);
    byte[] bytes = byteArrayOutputStream.toByteArray();
    list.add(bytes);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  return InternalRow.apply(asScalaBuffer(list).toList());
}
 
Example #13
Source File: LengthPrefixUnknownCodersTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Test bypassing unknown coders that are already wrapped with {@code LengthPrefixCoder} */
@Test
public void testLengthPrefixForLengthPrefixCoder() throws Exception {
  Coder<WindowedValue<KV<String, Integer>>> windowedValueCoder =
      WindowedValue.getFullCoder(
          KvCoder.of(StringUtf8Coder.of(), LengthPrefixCoder.of(VarIntCoder.of())),
          GlobalWindow.Coder.INSTANCE);

  Map<String, Object> lengthPrefixedCoderCloudObject =
      forCodec(CloudObjects.asCloudObject(windowedValueCoder, /*sdkComponents=*/ null), false);

  Coder<WindowedValue<KV<String, Integer>>> expectedCoder =
      WindowedValue.getFullCoder(
          KvCoder.of(StringUtf8Coder.of(), LengthPrefixCoder.of(VarIntCoder.of())),
          GlobalWindow.Coder.INSTANCE);

  assertEqualsAsJson(
      CloudObjects.asCloudObject(expectedCoder, /*sdkComponents=*/ null),
      lengthPrefixedCoderCloudObject);
}
 
Example #14
Source File: WriteFiles.java    From beam with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void process(ProcessContext c) throws Exception {
  getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
  @Nullable Integer fixedNumShards;
  if (numShardsView != null) {
    fixedNumShards = c.sideInput(numShardsView);
  } else if (getNumShardsProvider() != null) {
    fixedNumShards = getNumShardsProvider().get();
  } else {
    fixedNumShards = null;
  }
  List<FileResult<DestinationT>> fileResults = Lists.newArrayList(c.element());
  LOG.info("Finalizing {} file results", fileResults.size());
  DestinationT defaultDest = getDynamicDestinations().getDefaultDestination();
  List<KV<FileResult<DestinationT>, ResourceId>> resultsToFinalFilenames =
      fileResults.isEmpty()
          ? writeOperation.finalizeDestination(
              defaultDest, GlobalWindow.INSTANCE, fixedNumShards, fileResults)
          : finalizeAllDestinations(fileResults, fixedNumShards);
  writeOperation.moveToOutputFiles(resultsToFinalFilenames);
  for (KV<FileResult<DestinationT>, ResourceId> entry : resultsToFinalFilenames) {
    FileResult<DestinationT> res = entry.getKey();
    c.output(KV.of(res.getDestination(), entry.getValue().toString()));
  }
}
 
Example #15
Source File: StatefulTeamScoreTest.java    From deployment-examples with MIT License 6 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for one team.
 */
@Test
public void testScoreUpdatesOneTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)),
              event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)),
              event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)),
              event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)),
              event(TestUser.RED_TWO, 201, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401));

  p.run().waitUntilFinish();
}
 
Example #16
Source File: LateDataUtilsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(-100));
  assertThat(
      window.maxTimestamp().plus(allowedLateness),
      Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 
Example #17
Source File: PaneExtractorsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void nonLatePanesSingleEarly() {
  SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
      PaneExtractors.nonLatePanes();
  Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
      ImmutableList.of(
          ValueInSingleWindow.of(
              8,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(true, false, Timing.EARLY)),
          ValueInSingleWindow.of(
              4,
              new Instant(0L),
              GlobalWindow.INSTANCE,
              PaneInfo.createPane(true, false, Timing.EARLY)));

  assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8));
}
 
Example #18
Source File: WindowEvaluatorFactoryTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<BoundedWindow> assignWindows(AssignContext c) throws Exception {
  if (c.window().equals(GlobalWindow.INSTANCE)) {
    return Collections.singleton(new IntervalWindow(c.timestamp(), c.timestamp().plus(1L)));
  }
  return Collections.singleton(c.window());
}
 
Example #19
Source File: WatermarkCallbackExecutorTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void onGuaranteedFiringFiresAfterTrigger() throws Exception {
  CountDownLatch latch = new CountDownLatch(1);
  executor.callOnGuaranteedFiring(
      create,
      GlobalWindow.INSTANCE,
      WindowingStrategy.globalDefault(),
      new CountDownLatchCallback(latch));

  executor.fireForWatermark(create, BoundedWindow.TIMESTAMP_MAX_VALUE);
  assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true));
}
 
Example #20
Source File: ExecutableStageDoFnOperatorTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testEnsureStateCleanupWithKeyedInputCleanupTimer() {
  InMemoryTimerInternals inMemoryTimerInternals = new InMemoryTimerInternals();
  KeyedStateBackend keyedStateBackend = Mockito.mock(KeyedStateBackend.class);
  Lock stateBackendLock = Mockito.mock(Lock.class);
  StringUtf8Coder keyCoder = StringUtf8Coder.of();
  GlobalWindow window = GlobalWindow.INSTANCE;
  GlobalWindow.Coder windowCoder = GlobalWindow.Coder.INSTANCE;

  // Test that cleanup timer is set correctly
  ExecutableStageDoFnOperator.CleanupTimer cleanupTimer =
      new ExecutableStageDoFnOperator.CleanupTimer<>(
          inMemoryTimerInternals,
          stateBackendLock,
          WindowingStrategy.globalDefault(),
          keyCoder,
          windowCoder,
          keyedStateBackend);
  cleanupTimer.setForWindow(KV.of("key", "string"), window);

  Mockito.verify(stateBackendLock).lock();
  ByteBuffer key = FlinkKeyUtils.encodeKey("key", keyCoder);
  Mockito.verify(keyedStateBackend).setCurrentKey(key);
  assertThat(
      inMemoryTimerInternals.getNextTimer(TimeDomain.EVENT_TIME),
      is(window.maxTimestamp().plus(1)));
  Mockito.verify(stateBackendLock).unlock();
}
 
Example #21
Source File: AbstractPythonStatelessFunctionRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
private RunnerApi.WireCoderSetting createValueOnlyWireCoderSetting() throws IOException {
	WindowedValue<byte[]> value = WindowedValue.valueInGlobalWindow(new byte[0]);
	Coder<? extends BoundedWindow> windowCoder = GlobalWindow.Coder.INSTANCE;
	WindowedValue.FullWindowedValueCoder<byte[]> windowedValueCoder =
		WindowedValue.FullWindowedValueCoder.of(ByteArrayCoder.of(), windowCoder);
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	windowedValueCoder.encode(value, baos);
	return RunnerApi.WireCoderSetting.newBuilder()
		.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
		.setPayload(
			org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.ByteString.copyFrom(baos.toByteArray()))
		.build();
}
 
Example #22
Source File: SafeBrowsingTransforms.java    From nomulus with Apache License 2.0 5 votes vote down vote up
/** Evaluates any buffered {@link Subdomain} objects upon completing the bundle. */
@FinishBundle
public void finishBundle(FinishBundleContext context) {
  if (!subdomainBuffer.isEmpty()) {
    ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
    results.forEach((kv) -> context.output(kv, Instant.now(), GlobalWindow.INSTANCE));
  }
}
 
Example #23
Source File: PaneExtractorsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void onlyPaneOnlyOneFiring() {
  SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
      PaneExtractors.onlyPane(PAssert.PAssertionSite.capture(""));
  Iterable<ValueInSingleWindow<Integer>> onlyFiring =
      ImmutableList.of(
          ValueInSingleWindow.of(
              2, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING),
          ValueInSingleWindow.of(
              1, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING));

  assertThat(extractor.apply(onlyFiring), containsInAnyOrder(2, 1));
}
 
Example #24
Source File: SimpleDoFnRunnerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testOnTimerExceptionsWrappedAsUserCodeException() {
  ThrowingDoFn fn = new ThrowingDoFn();
  DoFnRunner<String, String> runner =
      new SimpleDoFnRunner<>(
          null,
          fn,
          NullSideInputReader.empty(),
          null,
          null,
          Collections.emptyList(),
          mockStepContext,
          null,
          Collections.emptyMap(),
          WindowingStrategy.of(new GlobalWindows()),
          DoFnSchemaInformation.create(),
          Collections.emptyMap());

  thrown.expect(UserCodeException.class);
  thrown.expectCause(is(fn.exceptionToThrow));

  runner.onTimer(
      TimerDeclaration.PREFIX + ThrowingDoFn.TIMER_ID,
      "",
      null,
      GlobalWindow.INSTANCE,
      new Instant(0),
      new Instant(0),
      TimeDomain.EVENT_TIME);
}
 
Example #25
Source File: ValueAndCoderLazySerializableTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void serializableAccumulatorKryoTest() {
  Iterable<WindowedValue<Integer>> accumulatedValue =
      Arrays.asList(winVal(0), winVal(1), winVal(3), winVal(4));

  final WindowedValue.FullWindowedValueCoder<Integer> wvaCoder =
      WindowedValue.FullWindowedValueCoder.of(
          BigEndianIntegerCoder.of(), GlobalWindow.Coder.INSTANCE);

  final IterableCoder<WindowedValue<Integer>> iterAccumCoder = IterableCoder.of(wvaCoder);

  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> accUnderTest =
      ValueAndCoderLazySerializable.of(accumulatedValue, iterAccumCoder);

  ValueAndCoderKryoSerializer kryoSerializer = new ValueAndCoderKryoSerializer();
  Kryo kryo = new Kryo();
  kryo.register(ValueAndCoderLazySerializable.class, kryoSerializer);

  ByteArrayOutputStream inMemOut = new ByteArrayOutputStream();
  Output out = new Output(inMemOut);
  kryo.writeObject(out, accUnderTest);
  out.close();

  Input input = new Input(new ByteArrayInputStream(inMemOut.toByteArray()));

  @SuppressWarnings("unchecked")
  ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>> materialized =
      (ValueAndCoderLazySerializable<Iterable<WindowedValue<Integer>>>)
          kryo.readObject(input, ValueAndCoderLazySerializable.class);
  input.close();

  assertEquals(accumulatedValue, materialized.getOrDecode(iterAccumCoder));
}
 
Example #26
Source File: ExecutableStageDoFnOperatorTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link ExecutableStageDoFnOperator}. Sets the runtime context to {@link
 * #runtimeContext}. The context factory is mocked to return {@link #stageContext} every time. The
 * behavior of the stage context itself is unchanged.
 */
@SuppressWarnings("rawtypes")
private ExecutableStageDoFnOperator getOperator(
    TupleTag<Integer> mainOutput,
    List<TupleTag<?>> additionalOutputs,
    DoFnOperator.MultiOutputOutputManagerFactory<Integer> outputManagerFactory) {
  return getOperator(
      mainOutput,
      additionalOutputs,
      outputManagerFactory,
      WindowingStrategy.globalDefault(),
      null,
      WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE));
}
 
Example #27
Source File: KeyedPValueTrackingVisitorTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void unkeyedInputWithKeyPreserving() {

  PCollection<KV<String, Iterable<WindowedValue<KV<String, Integer>>>>> input =
      p.apply(
          Create.of(
                  KV.of(
                      "hello",
                      (Iterable<WindowedValue<KV<String, Integer>>>)
                          Collections.<WindowedValue<KV<String, Integer>>>emptyList()))
              .withCoder(
                  KvCoder.of(
                      StringUtf8Coder.of(),
                      IterableCoder.of(
                          WindowedValue.getValueOnlyCoder(
                              KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))))));

  PCollection<KeyedWorkItem<String, KV<String, Integer>>> unkeyed =
      input
          .apply(ParDo.of(new ParDoMultiOverrideFactory.ToKeyedWorkItem<>()))
          .setCoder(
              KeyedWorkItemCoder.of(
                  StringUtf8Coder.of(),
                  KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()),
                  GlobalWindow.Coder.INSTANCE));

  p.traverseTopologically(visitor);
  assertThat(visitor.getKeyedPValues(), not(hasItem(unkeyed)));
}
 
Example #28
Source File: CoGbkResultCoderTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testCoderIsSerializableWithWellKnownCoderType() {
  CoderProperties.coderSerializable(
      CoGbkResultCoder.of(
          CoGbkResultSchema.of(ImmutableList.of(new TupleTag<GlobalWindow>())),
          UnionCoder.of(ImmutableList.of(GlobalWindow.Coder.INSTANCE))));
}
 
Example #29
Source File: ReifyTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void windowsInValueSucceeds() {
  PCollection<KV<String, Integer>> timestamped =
      pipeline
          .apply(Create.of(KV.of("foo", 0), KV.of("foo", 1), KV.of("bar", 2), KV.of("baz", 3)))
          .apply(TIMESTAMP_FROM_V);

  PCollection<KV<String, ValueInSingleWindow<Integer>>> reified =
      timestamped.apply(Reify.windowsInValue());

  PAssert.that(reified)
      .containsInAnyOrder(
          KV.of(
              "foo",
              ValueInSingleWindow.of(
                  0, new Instant(0), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)),
          KV.of(
              "foo",
              ValueInSingleWindow.of(
                  1, new Instant(1), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)),
          KV.of(
              "bar",
              ValueInSingleWindow.of(
                  2, new Instant(2), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)),
          KV.of(
              "baz",
              ValueInSingleWindow.of(
                  3, new Instant(3), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)));

  pipeline.run();
}
 
Example #30
Source File: TimerTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimerCoderWithInconsistentWithEqualsComponentCoders() throws Exception {
  Coder<Timer<String>> coder = Timer.Coder.of(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE);
  CoderProperties.coderSerializable(coder);
  CoderProperties.structuralValueDecodeEncodeEqual(
      coder,
      Timer.of(
          "key",
          "tag",
          Collections.singleton(GlobalWindow.INSTANCE),
          FIRE_TIME,
          HOLD_TIME,
          PaneInfo.NO_FIRING));
  CoderProperties.structuralValueDecodeEncodeEqual(
      coder, Timer.cleared("key", "tag", Collections.singleton(GlobalWindow.INSTANCE)));
  CoderProperties.structuralValueConsistentWithEquals(
      coder,
      Timer.of(
          "key",
          "tag",
          Collections.singleton(GlobalWindow.INSTANCE),
          FIRE_TIME,
          HOLD_TIME,
          PaneInfo.NO_FIRING),
      Timer.of(
          "key",
          "tag",
          Collections.singleton(GlobalWindow.INSTANCE),
          FIRE_TIME,
          HOLD_TIME,
          PaneInfo.NO_FIRING));
  CoderProperties.structuralValueConsistentWithEquals(
      coder,
      Timer.cleared("key", "tag", Collections.singleton(GlobalWindow.INSTANCE)),
      Timer.cleared("key", "tag", Collections.singleton(GlobalWindow.INSTANCE)));
}