Java Code Examples for org.apache.beam.sdk.coders.Coder#verifyDeterministic()

@Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
  applicableTo(input);

  // Verify that the input Coder<KV<K, V>> is a KvCoder<K, V>, and that
  // the key coder is deterministic.
  Coder<K> keyCoder = getKeyCoder(input.getCoder());
  try {
    keyCoder.verifyDeterministic();
  } catch (NonDeterministicException e) {
    throw new IllegalStateException("the keyCoder of a GroupByKey must be deterministic", e);
  }

  // This primitive operation groups by the combination of key and window,
  // merging windows as needed, using the windows assigned to the
  // key/value input elements and the window merge operation of the
  // window function associated with the input PCollection.
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      updateWindowingStrategy(input.getWindowingStrategy()),
      input.isBounded(),
      getOutputKvCoder(input.getCoder()));
}
 

/**
 * Returns a {@link CountMinSketchFn} combiner with the given input coder. <br>
 * <b>Warning :</b> the coder must be deterministic.
 *
 * @param coder the coder that encodes the elements' type
 */
public static <InputT> CountMinSketchFn<InputT> create(Coder<InputT> coder) {
  try {
    coder.verifyDeterministic();
  } catch (Coder.NonDeterministicException e) {
    throw new IllegalArgumentException(
        "Coder must be deterministic to perform this sketch." + e.getMessage(), e);
  }
  return new CountMinSketchFn<>(coder, 0.01, 0.999);
}
 

/**
 * Returns an {@link ApproximateDistinctFn} combiner with the given input coder.
 *
 * @param coder the coder that encodes the elements' type
 */
public static <InputT> ApproximateDistinctFn<InputT> create(Coder<InputT> coder) {
  try {
    coder.verifyDeterministic();
  } catch (Coder.NonDeterministicException e) {
    throw new IllegalArgumentException(
        "Coder must be deterministic to perform this sketch." + e.getMessage(), e);
  }
  return new ApproximateDistinctFn<>(12, 0, coder);
}
 

@Test
public void testNonDeterministicCoder() throws NonDeterministicException {
  thrown.expect(NonDeterministicException.class);
  thrown.expectMessage(MessageWithMap.class.getName() + " transitively includes Map field");

  Coder<MessageWithMap> coder = ProtoCoder.of(MessageWithMap.class);
  coder.verifyDeterministic();
}
 

/**
 * Verifies that for the given {@code Coder<T>}, {@code Coder.Context}, and values of type {@code
 * T}, if the values are equal then the encoded bytes are equal.
 */
public static <T> void coderDeterministicInContext(
    Coder<T> coder, Coder.Context context, T value1, T value2) throws Exception {
  try {
    coder.verifyDeterministic();
  } catch (NonDeterministicException e) {
    throw new AssertionError("Expected that the coder is deterministic", e);
  }
  assertThat("Expected that the passed in values are equal()", value1, equalTo(value2));
  assertThat(encode(coder, context, value1), equalTo(encode(coder, context, value2)));
}
 

private static boolean isKeyAndWindowCoderConsistentWithEquals(
    Coder<?> keyCoder, Coder<?> windowCoder) {
  try {
    keyCoder.verifyDeterministic();
    windowCoder.verifyDeterministic();
    return keyCoder.consistentWithEquals() && windowCoder.consistentWithEquals();
  } catch (Coder.NonDeterministicException ex) {
    throw new IllegalArgumentException(
        "Coder for both key " + keyCoder + " and " + windowCoder + " must be deterministic", ex);
  }
}
 

/** Validates that the key portion of the given coder is deterministic. */
public static void validateCoderIsCompatible(IsmRecordCoder<?> coder) {
  for (Coder<?> keyComponentCoder : coder.getKeyComponentCoders()) {
    try {
      keyComponentCoder.verifyDeterministic();
    } catch (NonDeterministicException e) {
      throw new IllegalArgumentException(
          String.format(
              "Key component coder %s is expected to be deterministic.", keyComponentCoder),
          e);
    }
  }
}
 

@Test
public void testDeterministicCoder() throws NonDeterministicException {
  Coder<MessageA> coder = ProtoCoder.of(MessageA.class);
  coder.verifyDeterministic();
}
 

@Override
public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
  if (input.isBounded() == IsBounded.UNBOUNDED) {
    checkArgument(
        getWindowedWrites(),
        "Must use windowed writes when applying %s to an unbounded PCollection",
        WriteFiles.class.getSimpleName());
    // The reason for this is https://issues.apache.org/jira/browse/BEAM-1438
    // and similar behavior in other runners.
    checkArgument(
        getComputeNumShards() != null || getNumShardsProvider() != null,
        "When applying %s to an unbounded PCollection, "
            + "must specify number of output shards explicitly",
        WriteFiles.class.getSimpleName());
  }
  this.writeOperation = getSink().createWriteOperation();
  this.writeOperation.setWindowedWrites(getWindowedWrites());

  if (!getWindowedWrites()) {
    // Re-window the data into the global window and remove any existing triggers.
    input =
        input.apply(
            "RewindowIntoGlobal",
            Window.<UserT>into(new GlobalWindows())
                .triggering(DefaultTrigger.of())
                .discardingFiredPanes());
  }

  Coder<DestinationT> destinationCoder;
  try {
    destinationCoder =
        getDynamicDestinations()
            .getDestinationCoderWithDefault(input.getPipeline().getCoderRegistry());
    destinationCoder.verifyDeterministic();
  } catch (CannotProvideCoderException | NonDeterministicException e) {
    throw new RuntimeException(e);
  }
  @SuppressWarnings("unchecked")
  Coder<BoundedWindow> windowCoder =
      (Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder();
  FileResultCoder<DestinationT> fileResultCoder =
      FileResultCoder.of(windowCoder, destinationCoder);

  PCollectionView<Integer> numShardsView =
      (getComputeNumShards() == null) ? null : input.apply(getComputeNumShards());

  PCollection<FileResult<DestinationT>> tempFileResults =
      (getComputeNumShards() == null && getNumShardsProvider() == null)
          ? input.apply(
              "WriteUnshardedBundlesToTempFiles",
              new WriteUnshardedBundlesToTempFiles(destinationCoder, fileResultCoder))
          : input.apply(
              "WriteShardedBundlesToTempFiles",
              new WriteShardedBundlesToTempFiles(
                  destinationCoder, fileResultCoder, numShardsView));

  return tempFileResults
      .apply("GatherTempFileResults", new GatherResults<>(fileResultCoder))
      .apply(
          "FinalizeTempFileBundles",
          new FinalizeTempFileBundles(numShardsView, destinationCoder));
}