Java Code Examples for org.joda.time.Duration#millis()

/**
 * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using
 * the provided components to dereferences identifiers found in the proto.
 */
public static WindowingStrategy<?, ?> fromProto(
    RunnerApi.WindowingStrategy proto, RehydratedComponents components)
    throws InvalidProtocolBufferException {

  FunctionSpec windowFnSpec = proto.getWindowFn();
  WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec);
  TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime());
  AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode());
  Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger());
  ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior());
  Duration allowedLateness = Duration.millis(proto.getAllowedLateness());
  OnTimeBehavior onTimeBehavior = fromProto(proto.getOnTimeBehavior());

  return WindowingStrategy.of(windowFn)
      .withAllowedLateness(allowedLateness)
      .withMode(accumulationMode)
      .withTrigger(trigger)
      .withTimestampCombiner(timestampCombiner)
      .withClosingBehavior(closingBehavior)
      .withOnTimeBehavior(onTimeBehavior);
}
 

public RedisClusterIngestionClient(StoreProto.Store.RedisClusterConfig redisClusterConfig) {
  this.uriList =
      Arrays.stream(redisClusterConfig.getConnectionString().split(","))
          .map(
              hostPort -> {
                String[] hostPortSplit = hostPort.trim().split(":");
                return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1]));
              })
          .collect(Collectors.toList());

  long backoffMs =
      redisClusterConfig.getInitialBackoffMs() > 0 ? redisClusterConfig.getInitialBackoffMs() : 1;
  this.backOffExecutor =
      new BackOffExecutor(redisClusterConfig.getMaxRetries(), Duration.millis(backoffMs));
  this.clusterClient = RedisClusterClient.create(uriList);
}
 

@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(-100));
  assertThat(
      window.maxTimestamp().plus(allowedLateness),
      Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 

@VisibleForTesting
static Duration resolveDefaultApiTimeout(Read<?, ?> spec) {

  // KIP-266 - let's allow to configure timeout in consumer settings. This is supported in
  // higher versions of kafka client. We allow users to set this timeout and it will be
  // respected
  // in all places where Beam's KafkaIO handles possibility of API call being blocked.
  // Later, we should replace the string with ConsumerConfig constant
  Duration timeout =
      tryParseDurationFromMillis(spec.getConsumerConfig().get("default.api.timeout.ms"));
  if (timeout == null) {
    Duration value =
        tryParseDurationFromMillis(
            spec.getConsumerConfig().get(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG));
    if (value != null) {
      // 2x request timeout to be compatible with previous version
      timeout = Duration.millis(2 * value.getMillis());
    }
  }

  return timeout == null ? Duration.standardSeconds(60) : timeout;
}
 

public RedisStandaloneIngestionClient(StoreProto.Store.RedisConfig redisConfig) {
  this.host = redisConfig.getHost();
  this.port = redisConfig.getPort();
  long backoffMs = redisConfig.getInitialBackoffMs() > 0 ? redisConfig.getInitialBackoffMs() : 1;
  this.backOffExecutor =
      new BackOffExecutor(redisConfig.getMaxRetries(), Duration.millis(backoffMs));
}
 

private static Duration tryParseDurationFromMillis(Object value) {
  if (value == null) {
    return null;
  }
  return value instanceof Integer
      ? Duration.millis((Integer) value)
      : Duration.millis(Integer.parseInt(value.toString()));
}
 

private synchronized void initializeLocalHost(int port) throws IOException {
  this.logEveryNStreamFailures = 1;
  this.maxBackoff = Duration.millis(500);
  this.unaryDeadlineSeconds = 10; // For local testing use short deadlines.
  Channel channel = localhostChannel(port);
  if (streamingEngineEnabled()) {
    this.stubList.add(CloudWindmillServiceV1Alpha1Grpc.newStub(channel));
    this.syncStubList.add(CloudWindmillServiceV1Alpha1Grpc.newBlockingStub(channel));
  } else {
    this.syncApplianceStub = WindmillApplianceGrpc.newBlockingStub(channel);
  }
}
 

public Duration getQueryTimeout()
{
    if (queryTimeout == null) {
        io.dropwizard.util.Duration duration = io.dropwizard.util.Duration.parse(timeout);
        queryTimeout = Duration.millis(duration.toMilliseconds());
    }

    return queryTimeout;
}
 

private KV<byte[], byte[]> outputElement(
    byte[] inputKey, byte[] inputValue, long inputValueHashcode, int index, Random random) {

  long seed = options.hashFunction().hashLong(inputValueHashcode + index).asLong();
  Duration delay = Duration.millis(options.nextDelay(seed));
  long millisecondsSpentSleeping = 0;

  while (delay.getMillis() > 0) {
    millisecondsSpentSleeping +=
        delay(delay, options.cpuUtilizationInMixedDelay, options.delayType, random);

    if (isWithinThroughputLimit()) {
      break;
    } else {
      // try an extra delay of 1 millisecond
      delay = Duration.millis(1);
    }
  }

  reportThrottlingTimeMetrics(millisecondsSpentSleeping);

  if (options.preservesInputKeyDistribution) {
    // Generate the new byte array value whose hashcode will be
    // used as seed to initialize a Random object in next stages.
    byte[] newValue = new byte[inputValue.length];
    random.nextBytes(newValue);
    return KV.of(inputKey, newValue);
  } else {
    return options.genKvPair(seed);
  }
}
 

@Test
public void windowOnlyContextThrowsOnSideInput() {
  BoundedWindow window = new IntervalWindow(new Instant(-137), Duration.millis(21L));
  StateContext<BoundedWindow> context = StateContexts.windowOnlyContext(window);
  thrown.expect(IllegalArgumentException.class);
  context.sideInput(view);
}
 

@Test
public void windowOnlyContextThrowsOnOptions() {
  BoundedWindow window = new IntervalWindow(new Instant(-137), Duration.millis(21L));
  StateContext<BoundedWindow> context = StateContexts.windowOnlyContext(window);
  thrown.expect(IllegalArgumentException.class);
  context.getPipelineOptions();
}
 

@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testDiscardingMode() {
  TestStream<String> stream =
      TestStream.create(StringUtf8Coder.of())
          .advanceWatermarkTo(new Instant(0))
          .addElements(
              TimestampedValue.of("firstPane", new Instant(100)),
              TimestampedValue.of("alsoFirstPane", new Instant(200)))
          .addElements(TimestampedValue.of("onTimePane", new Instant(500)))
          .advanceWatermarkTo(new Instant(1000L))
          .addElements(
              TimestampedValue.of("finalLatePane", new Instant(750)),
              TimestampedValue.of("alsoFinalLatePane", new Instant(250)))
          .advanceWatermarkToInfinity();

  FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
  Duration allowedLateness = Duration.millis(5000L);
  PCollection<String> values =
      p.apply(stream)
          .apply(
              Window.<String>into(windowFn)
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(2))
                          .withLateFirings(Never.ever()))
                  .discardingFiredPanes()
                  .withAllowedLateness(allowedLateness))
          .apply(WithKeys.of(1))
          .apply(GroupByKey.create())
          .apply(Values.create())
          .apply(Flatten.iterables());

  IntervalWindow window = windowFn.assignWindow(new Instant(100));
  PAssert.that(values)
      .inWindow(window)
      .containsInAnyOrder(
          "firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
  PAssert.that(values)
      .inCombinedNonLatePanes(window)
      .containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
  PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
  PAssert.that(values)
      .inFinalPane(window)
      .containsInAnyOrder("finalLatePane", "alsoFinalLatePane");

  p.run();
}
 

public static Duration convert(com.google.protobuf.Duration d) {
  return Duration.millis(Durations.toMillis(d));
}
 

@Test
public void testLateDroppingForStatefulFn() throws Exception {

  WindowingStrategy<Object, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(FixedWindows.of(new Duration(10)));

  DoFn<Integer, String> fn =
      new DoFn<Integer, String>() {

        @StateId("state")
        private final StateSpec<ValueState<String>> stateSpec =
            StateSpecs.value(StringUtf8Coder.of());

        @ProcessElement
        public void processElement(ProcessContext context) {
          context.output(context.element().toString());
        }
      };

  VarIntCoder keyCoder = VarIntCoder.of();
  Coder<WindowedValue<Integer>> inputCoder =
      WindowedValue.getFullCoder(keyCoder, windowingStrategy.getWindowFn().windowCoder());
  Coder<WindowedValue<String>> outputCoder =
      WindowedValue.getFullCoder(
          StringUtf8Coder.of(), windowingStrategy.getWindowFn().windowCoder());

  KeySelector<WindowedValue<Integer>, ByteBuffer> keySelector =
      e -> FlinkKeyUtils.encodeKey(e.getValue(), keyCoder);

  TupleTag<String> outputTag = new TupleTag<>("main-output");

  DoFnOperator<Integer, String> doFnOperator =
      new DoFnOperator<>(
          fn,
          "stepName",
          inputCoder,
          Collections.emptyMap(),
          outputTag,
          Collections.emptyList(),
          new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, outputCoder),
          windowingStrategy,
          new HashMap<>(), /* side-input mapping */
          Collections.emptyList(), /* side inputs */
          PipelineOptionsFactory.as(FlinkPipelineOptions.class),
          keyCoder, /* key coder */
          keySelector,
          DoFnSchemaInformation.create(),
          Collections.emptyMap());

  OneInputStreamOperatorTestHarness<WindowedValue<Integer>, WindowedValue<String>> testHarness =
      new KeyedOneInputStreamOperatorTestHarness<>(
          doFnOperator,
          keySelector,
          new CoderTypeInformation<>(FlinkKeyUtils.ByteBufferCoder.of()));

  testHarness.open();

  testHarness.processWatermark(0);

  IntervalWindow window1 = new IntervalWindow(new Instant(0), Duration.millis(10));

  // this should not be late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(13, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      contains(WindowedValue.of("13", new Instant(0), window1, PaneInfo.NO_FIRING)));

  testHarness.getOutput().clear();

  testHarness.processWatermark(9);

  // this should still not be considered late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(17, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      contains(WindowedValue.of("17", new Instant(0), window1, PaneInfo.NO_FIRING)));

  testHarness.getOutput().clear();

  testHarness.processWatermark(10);

  // this should now be considered late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(17, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(stripStreamRecordFromWindowedValue(testHarness.getOutput()), emptyIterable());

  testHarness.close();
}
 

/**
 * Tests that the garbage collection time for a fixed window does not overflow the end of time.
 */
@Test
public void testFixedWindowEndOfTimeGarbageCollection() throws Exception {
  Duration allowedLateness = Duration.standardDays(365);
  Duration windowSize = Duration.millis(10);
  WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(windowSize);

  // This timestamp falls into a window where the end of the window is before the end of the
  // global window - the "end of time" - yet its expiration time is after.
  final Instant elementTimestamp =
      GlobalWindow.INSTANCE.maxTimestamp().minus(allowedLateness).plus(1);

  IntervalWindow window =
      Iterables.getOnlyElement(
          windowFn.assignWindows(
              windowFn.new AssignContext() {
                @Override
                public Object element() {
                  throw new UnsupportedOperationException();
                }

                @Override
                public Instant timestamp() {
                  return elementTimestamp;
                }

                @Override
                public BoundedWindow window() {
                  throw new UnsupportedOperationException();
                }
              }));

  assertTrue(window.maxTimestamp().isBefore(GlobalWindow.INSTANCE.maxTimestamp()));
  assertTrue(
      window.maxTimestamp().plus(allowedLateness).isAfter(GlobalWindow.INSTANCE.maxTimestamp()));

  // Test basic execution of a trigger using a non-combining window set and accumulating mode.

  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) windowFn)
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(Never.ever()))
          .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .withAllowedLateness(allowedLateness);

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.injectElements(TimestampedValue.of(13, elementTimestamp));

  // Should fire ON_TIME pane and there will be a checkState that the cleanup time
  // is prior to timestamp max value
  tester.advanceInputWatermark(window.maxTimestamp());

  // Nothing in the ON_TIME pane (not governed by triggers, but by ReduceFnRunner)
  assertThat(tester.extractOutput(), emptyIterable());

  tester.injectElements(TimestampedValue.of(42, elementTimestamp));

  // Now the final pane should fire, demonstrating that the GC time was truncated
  tester.advanceInputWatermark(GlobalWindow.INSTANCE.maxTimestamp());
  assertThat(tester.extractOutput(), contains(isWindowedValue(equalTo(55))));
}
 

@Test
public void testTimerCleanupOfPendingTimerList() throws Exception {
  // test harness
  WindowDoFnOperator<Long, Long, Long> windowDoFnOperator = getWindowDoFnOperator();
  KeyedOneInputStreamOperatorTestHarness<
          ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>>
      testHarness = createTestHarness(windowDoFnOperator);
  testHarness.open();

  DoFnOperator<KeyedWorkItem<Long, Long>, KV<Long, Long>>.FlinkTimerInternals timerInternals =
      windowDoFnOperator.timerInternals;

  // process elements
  IntervalWindow window = new IntervalWindow(new Instant(0), Duration.millis(100));
  IntervalWindow window2 = new IntervalWindow(new Instant(100), Duration.millis(100));
  testHarness.processWatermark(0L);

  // Use two different keys to check for correct watermark hold calculation
  testHarness.processElement(
      Item.builder().key(1L).timestamp(1L).value(100L).window(window).build().toStreamRecord());
  testHarness.processElement(
      Item.builder()
          .key(2L)
          .timestamp(150L)
          .value(150L)
          .window(window2)
          .build()
          .toStreamRecord());

  testHarness.processWatermark(1);

  // Note that the following is 1 because the state is key-partitioned
  assertThat(Iterables.size(timerInternals.pendingTimersById.keys()), is(1));

  assertThat(testHarness.numKeyedStateEntries(), is(6));
  assertThat(windowDoFnOperator.getCurrentOutputWatermark(), is(1L));
  assertThat(timerInternals.getMinOutputTimestampMs(), is(Long.MAX_VALUE));

  // close window
  testHarness.processWatermark(100L);

  // Note that the following is zero because we only the first key is active
  assertThat(Iterables.size(timerInternals.pendingTimersById.keys()), is(0));

  assertThat(testHarness.numKeyedStateEntries(), is(3));
  assertThat(windowDoFnOperator.getCurrentOutputWatermark(), is(100L));
  assertThat(timerInternals.getMinOutputTimestampMs(), is(Long.MAX_VALUE));

  testHarness.processWatermark(200L);

  // All the state has been cleaned up
  assertThat(testHarness.numKeyedStateEntries(), is(0));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      containsInAnyOrder(
          WindowedValue.of(
              KV.of(1L, 100L), new Instant(99), window, PaneInfo.createPane(true, true, ON_TIME)),
          WindowedValue.of(
              KV.of(2L, 150L),
              new Instant(199),
              window2,
              PaneInfo.createPane(true, true, ON_TIME))));

  // cleanup
  testHarness.close();
}
 

@Test
public void testMergingWatermarkHoldLateNewWindowMerged() throws Exception {
  Duration allowedLateness = Duration.standardMinutes(1);
  Duration gapDuration = Duration.millis(10);
  ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(Sessions.withGapDuration(gapDuration))
              .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
              .withTrigger(
                  Repeatedly.forever(
                      AfterWatermark.pastEndOfWindow()
                          .withLateFirings(AfterPane.elementCountAtLeast(1))))
              .withAllowedLateness(allowedLateness));
  tester.setAutoAdvanceOutputWatermark(false);

  assertEquals(null, tester.getWatermarkHold());
  assertEquals(null, tester.getOutputWatermark());
  tester.advanceInputWatermark(new Instant(24));
  injectElements(tester, 1);
  assertThat(tester.getWatermarkHold(), nullValue());
  injectElements(tester, 14);
  assertThat(tester.getWatermarkHold(), nullValue());
  injectElements(tester, 6, 16);
  // There should now be a watermark hold since the window has extended past the input watermark.
  // The hold should be for the end of the window (last element + gapDuration - 1).
  assertEquals(tester.getWatermarkHold(), new Instant(25));
  injectElements(tester, 6, 21);
  // The hold should be extended with the window.
  assertEquals(tester.getWatermarkHold(), new Instant(30));
  // Advancing the watermark should remove the hold.
  tester.advanceInputWatermark(new Instant(31));
  assertThat(tester.getWatermarkHold(), nullValue());
  // Late elements added to the window should not generate a hold.
  injectElements(tester, 0);
  assertThat(tester.getWatermarkHold(), nullValue());
  // Generate a new window that is ontime.
  injectElements(tester, 32, 40);
  assertEquals(tester.getWatermarkHold(), new Instant(49));
  // Join the closed window with the new window.
  injectElements(tester, 24);
  assertEquals(tester.getWatermarkHold(), new Instant(49));
  tester.advanceInputWatermark(new Instant(50));
  assertThat(tester.getWatermarkHold(), nullValue());
}
 

/**
 * Tests that {@link SimpleDoFnRunner#onTimer} properly dispatches to the underlying {@link DoFn}.
 */
@Test
public void testOnTimerCalled() {
  WindowFn<?, GlobalWindow> windowFn = new GlobalWindows();
  DoFnWithTimers<GlobalWindow> fn = new DoFnWithTimers(windowFn.windowCoder());
  DoFnRunner<String, String> runner =
      new SimpleDoFnRunner<>(
          null,
          fn,
          NullSideInputReader.empty(),
          null,
          null,
          Collections.emptyList(),
          mockStepContext,
          null,
          Collections.emptyMap(),
          WindowingStrategy.of(windowFn),
          DoFnSchemaInformation.create(),
          Collections.emptyMap());

  Instant currentTime = new Instant(42);
  Duration offset = Duration.millis(37);

  // Mocking is not easily compatible with annotation analysis, so we manually record
  // the method call.
  runner.onTimer(
      TimerDeclaration.PREFIX + DoFnWithTimers.TIMER_ID,
      "",
      null,
      GlobalWindow.INSTANCE,
      currentTime.plus(offset),
      currentTime.plus(offset),
      TimeDomain.EVENT_TIME);

  assertThat(
      fn.onTimerInvocations,
      contains(
          TimerData.of(
              DoFnWithTimers.TIMER_ID,
              "",
              StateNamespaces.window(windowFn.windowCoder(), GlobalWindow.INSTANCE),
              currentTime.plus(offset),
              currentTime.plus(offset),
              TimeDomain.EVENT_TIME)));
}
 

public ClusterWait(ClusterHealthCheck clusterHealthCheck, ReadableDuration timeout) {
    this.clusterHealthCheck = clusterHealthCheck;
    this.timeout = Duration.millis(timeout.getMillis());
}
 

private static String print(long duration, TimeUnit timeUnit, PeriodFormatter formatter, boolean trim)
{
   Duration durationObject = Duration.millis(timeUnit.toMillis(duration));

   Period normalizedPeriod = durationObject.toPeriod().normalizedStandard();

   String durationString = formatter.print(normalizedPeriod);

   return trim ? durationString.trim() : durationString;
}