Java Code Examples for org.apache.beam.sdk.transforms.Sum#ofLongs()

/** {@link CombineFn} for Sum based on {@link Sum} and {@link Combine.BinaryCombineFn}. */
static CombineFn createSum(Schema.FieldType fieldType) {
  switch (fieldType.getTypeName()) {
    case INT32:
      return Sum.ofIntegers();
    case INT16:
      return new ShortSum();
    case BYTE:
      return new ByteSum();
    case INT64:
      return Sum.ofLongs();
    case FLOAT:
      return new FloatSum();
    case DOUBLE:
      return Sum.ofDoubles();
    case DECIMAL:
      return new BigDecimalSum();
    default:
      throw new UnsupportedOperationException(
          String.format("[%s] is not supported in SUM", fieldType));
  }
}
 

@VisibleForTesting
AdaptiveThrottler(long samplePeriodMs, long sampleUpdateMs, double overloadRatio, Random random) {
  allRequests =
      new MovingFunction(
          samplePeriodMs,
          sampleUpdateMs,
          1 /* numSignificantBuckets */,
          1 /* numSignificantSamples */,
          Sum.ofLongs());
  successfulRequests =
      new MovingFunction(
          samplePeriodMs,
          sampleUpdateMs,
          1 /* numSignificantBuckets */,
          1 /* numSignificantSamples */,
          Sum.ofLongs());
  this.overloadRatio = overloadRatio;
  this.random = random;
}
 

public MovingAverage(
    long samplePeriodMs,
    long sampleUpdateMs,
    int numSignificantBuckets,
    int numSignificantSamples) {
  sum =
      new MovingFunction(
          samplePeriodMs,
          sampleUpdateMs,
          numSignificantBuckets,
          numSignificantSamples,
          Sum.ofLongs());
  count =
      new MovingFunction(
          samplePeriodMs,
          sampleUpdateMs,
          numSignificantBuckets,
          numSignificantSamples,
          Sum.ofLongs());
}
 

@Test
public void testCombinesElementsInSlidingWindows() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();
  AppliedCombineFn<String, Long, ?, Long> appliedFn =
      AppliedCombineFn.withInputCoder(
          combineFn,
          CoderRegistry.createDefault(),
          KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));

  GroupAlsoByWindowProperties.combinesElementsInSlidingWindows(
      new CombiningGABWViaOutputBufferDoFnFactory<>(StringUtf8Coder.of(), appliedFn), combineFn);
}
 

@Test
public void testCombinesIntoSessions() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();
  AppliedCombineFn<String, Long, ?, Long> appliedFn =
      AppliedCombineFn.withInputCoder(
          combineFn,
          CoderRegistry.createDefault(),
          KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));

  GroupAlsoByWindowProperties.combinesElementsPerSession(
      new CombiningGABWViaOutputBufferDoFnFactory<>(StringUtf8Coder.of(), appliedFn), combineFn);
}
 

@Test
public void testCombinesIntoSessionsWithEndOfWindowTimestamp() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();
  AppliedCombineFn<String, Long, ?, Long> appliedFn =
      AppliedCombineFn.withInputCoder(
          combineFn,
          CoderRegistry.createDefault(),
          KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));

  GroupAlsoByWindowProperties.combinesElementsPerSessionWithEndOfWindowTimestamp(
      new CombiningGABWViaOutputBufferDoFnFactory<>(StringUtf8Coder.of(), appliedFn), combineFn);
}
 

@Test
public void testCombinesElementsInSlidingWindows() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();

  GroupAlsoByWindowProperties.combinesElementsInSlidingWindows(
      new GABWAndCombineDoFnFactory<>(combineFn), combineFn);
}
 

@Test
public void testCombinesIntoSessions() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();

  GroupAlsoByWindowProperties.combinesElementsPerSession(
      new GABWAndCombineDoFnFactory<>(combineFn), combineFn);
}
 

@Test
public void testCombinesIntoSessionsWithEndOfWindowTimestamp() throws Exception {
  CombineFn<Long, ?, Long> combineFn = Sum.ofLongs();

  GroupAlsoByWindowProperties.combinesElementsPerSessionWithEndOfWindowTimestamp(
      new GABWAndCombineDoFnFactory<>(combineFn), combineFn);
}
 

@Test
public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException {
  CopyOnAccessInMemoryStateInternals<String> underlying =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs();

  StateNamespace namespace = new StateNamespaceForTest("foo");
  CoderRegistry reg = pipeline.getCoderRegistry();
  StateTag<CombiningState<Long, long[], Long>> stateTag =
      StateTags.combiningValue(
          "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn);
  GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag);
  assertThat(underlyingValue.read(), equalTo(0L));

  underlyingValue.add(1L);
  assertThat(underlyingValue.read(), equalTo(1L));

  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
  GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag);
  assertThat(copyOnAccessState.read(), equalTo(1L));

  copyOnAccessState.add(4L);
  assertThat(copyOnAccessState.read(), equalTo(5L));
  assertThat(underlyingValue.read(), equalTo(1L));

  GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag);
  assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read()));
}
 

@Test
public void testMultipleAccumulatorsInMergingCombineFn() {
  BinaryCombineLongFn originalFn = Sum.ofLongs();
  MergingCombineFn<Void, long[]> fn =
      new MergingCombineFn<>(
          originalFn, originalFn.getAccumulatorCoder(p.getCoderRegistry(), VarLongCoder.of()));

  long[] inputAccum = originalFn.createAccumulator();
  inputAccum = originalFn.addInput(inputAccum, 1L);
  inputAccum = originalFn.addInput(inputAccum, 2L);
  assertThat(inputAccum[0], equalTo(3L));

  long[] otherAccum = originalFn.createAccumulator();
  otherAccum = originalFn.addInput(otherAccum, 4L);
  assertThat(otherAccum[0], equalTo(4L));

  List<long[]> first = fn.createAccumulator();
  first = fn.addInput(first, inputAccum);
  assertThat(first, hasItem(inputAccum));
  assertThat(inputAccum.length, equalTo(1));
  assertThat(inputAccum[0], equalTo(3L));

  List<long[]> second = fn.createAccumulator();
  second = fn.addInput(second, inputAccum);
  assertThat(second, hasItem(inputAccum));
  assertThat(inputAccum.length, equalTo(1));
  assertThat(inputAccum[0], equalTo(3L));

  List<long[]> mergeToSecond = fn.createAccumulator();
  mergeToSecond = fn.addInput(mergeToSecond, otherAccum);
  assertThat(mergeToSecond, hasItem(otherAccum));
  assertThat(otherAccum.length, equalTo(1));
  assertThat(otherAccum[0], equalTo(4L));

  List<long[]> firstSelfMerged = fn.mergeAccumulators(ImmutableList.of(first));
  List<long[]> compactedFirst = fn.compact(firstSelfMerged);
  assertThat(firstSelfMerged, equalTo(compactedFirst));
  assertThat(firstSelfMerged, hasSize(1));
  assertThat(firstSelfMerged.get(0)[0], equalTo(3L));

  List<long[]> secondMerged = fn.mergeAccumulators(ImmutableList.of(second, mergeToSecond));
  List<long[]> secondCompacted = fn.compact(secondMerged);
  assertThat(secondCompacted, hasSize(1));
  assertThat(secondCompacted.get(0)[0], equalTo(7L));
  assertThat(firstSelfMerged, equalTo(compactedFirst));
  assertThat(firstSelfMerged, hasSize(1));
  assertThat(firstSelfMerged.get(0)[0], equalTo(3L));
}