org.apache.beam.sdk.coders.CannotProvideCoderException Java Examples

The following examples show how to use org.apache.beam.sdk.coders.CannotProvideCoderException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DynamicProtoCoder.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public <T> Coder<T> coderFor(
    TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
    throws CannotProvideCoderException {
  if (!typeDescriptor.isSubtypeOf(MESSAGE_TYPE)) {
    throw new CannotProvideCoderException(
        String.format(
            "Cannot provide %s because %s is not a subclass of %s",
            DynamicProtoCoder.class.getSimpleName(), typeDescriptor, Message.class.getName()));
  }

  @SuppressWarnings("unchecked")
  TypeDescriptor<? extends Message> messageType =
      (TypeDescriptor<? extends Message>) typeDescriptor;
  try {
    @SuppressWarnings("unchecked")
    Coder<T> coder = (Coder<T>) DynamicProtoCoder.of(messageType);
    return coder;
  } catch (IllegalArgumentException e) {
    throw new CannotProvideCoderException(e);
  }
}
 
Example #2
Source File: FlattenTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
@Category({ValidatesRunner.class, FlattenWithHeterogeneousCoders.class})
public void testFlattenMultipleCoders() throws CannotProvideCoderException {
  PCollection<Long> bigEndianLongs =
      p.apply(
          "BigEndianLongs",
          Create.of(0L, 1L, 2L, 3L, null, 4L, 5L, null, 6L, 7L, 8L, null, 9L)
              .withCoder(NullableCoder.of(BigEndianLongCoder.of())));
  PCollection<Long> varLongs =
      p.apply("VarLengthLongs", GenerateSequence.from(0).to(5)).setCoder(VarLongCoder.of());

  PCollection<Long> flattened =
      PCollectionList.of(bigEndianLongs)
          .and(varLongs)
          .apply(Flatten.pCollections())
          .setCoder(NullableCoder.of(VarLongCoder.of()));
  PAssert.that(flattened)
      .containsInAnyOrder(
          0L, 0L, 1L, 1L, 2L, 3L, 2L, 4L, 5L, 3L, 6L, 7L, 4L, 8L, 9L, null, null, null);
  p.run();
}
 
Example #3
Source File: LatestFnTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testDefaultCoderHandlesNull() throws CannotProvideCoderException {
  Latest.LatestFn<Long> fn = new Latest.LatestFn<>();

  CoderRegistry registry = CoderRegistry.createDefault();
  TimestampedValue.TimestampedValueCoder<Long> inputCoder =
      TimestampedValue.TimestampedValueCoder.of(VarLongCoder.of());

  assertThat(
      "Default output coder should handle null values",
      fn.getDefaultOutputCoder(registry, inputCoder),
      instanceOf(NullableCoder.class));
  assertThat(
      "Default accumulator coder should handle null values",
      fn.getAccumulatorCoder(registry, inputCoder),
      instanceOf(NullableCoder.class));
}
 
Example #4
Source File: DynamicDestinations.java    From beam with Apache License 2.0 6 votes vote down vote up
Coder<DestinationT> getDestinationCoderWithDefault(CoderRegistry registry)
    throws CannotProvideCoderException {
  Coder<DestinationT> destinationCoder = getDestinationCoder();
  if (destinationCoder != null) {
    return destinationCoder;
  }
  // If dynamicDestinations doesn't provide a coder, try to find it in the coder registry.
  TypeDescriptor<DestinationT> descriptor =
      extractFromTypeParameters(
          this,
          DynamicDestinations.class,
          new TypeDescriptors.TypeVariableExtractor<
              DynamicDestinations<T, DestinationT>, DestinationT>() {});
  try {
    return registry.getCoder(descriptor);
  } catch (CannotProvideCoderException e) {
    throw new CannotProvideCoderException(
        "Failed to infer coder for DestinationT from type "
            + descriptor
            + ", please provide it explicitly by overriding getDestinationCoder()",
        e);
  }
}
 
Example #5
Source File: WritableCoder.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public <T> Coder<T> coderFor(
    TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
    throws CannotProvideCoderException {
  if (!typeDescriptor.isSubtypeOf(WRITABLE_TYPE)) {
    throw new CannotProvideCoderException(
        String.format(
            "Cannot provide %s because %s does not implement the interface %s",
            WritableCoder.class.getSimpleName(), typeDescriptor, Writable.class.getName()));
  }

  try {
    @SuppressWarnings("unchecked")
    Coder<T> coder = WritableCoder.of((Class) typeDescriptor.getRawType());
    return coder;
  } catch (IllegalArgumentException e) {
    throw new CannotProvideCoderException(e);
  }
}
 
Example #6
Source File: HBaseMutationCoder.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public <T> Coder<T> coderFor(
    TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
    throws CannotProvideCoderException {
  if (!typeDescriptor.isSubtypeOf(HBASE_MUTATION_TYPE_DESCRIPTOR)) {
    throw new CannotProvideCoderException(
        String.format(
            "Cannot provide %s because %s is not a subclass of %s",
            HBaseMutationCoder.class.getSimpleName(),
            typeDescriptor,
            Mutation.class.getName()));
  }

  try {
    @SuppressWarnings("unchecked")
    Coder<T> coder = (Coder<T>) HBaseMutationCoder.of();
    return coder;
  } catch (IllegalArgumentException e) {
    throw new CannotProvideCoderException(e);
  }
}
 
Example #7
Source File: FileIO.java    From beam with Apache License 2.0 6 votes vote down vote up
private Coder<DestinationT> resolveDestinationCoder(PCollection<UserT> input) {
  Coder<DestinationT> destinationCoder = getDestinationCoder();
  if (destinationCoder == null) {
    TypeDescriptor<DestinationT> destinationT =
        TypeDescriptors.outputOf(getDestinationFn().getClosure());
    try {
      destinationCoder = input.getPipeline().getCoderRegistry().getCoder(destinationT);
    } catch (CannotProvideCoderException e) {
      throw new IllegalArgumentException(
          "Unable to infer a coder for destination type (inferred from .by() as \""
              + destinationT
              + "\") - specify it explicitly using .withDestinationCoder()");
    }
  }
  return destinationCoder;
}
 
Example #8
Source File: AvroIO.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <T> Coder<T> inferCoder(
    @Nullable Coder<T> explicitCoder,
    SerializableFunction<GenericRecord, T> parseFn,
    CoderRegistry coderRegistry) {
  if (explicitCoder != null) {
    return explicitCoder;
  }
  // If a coder was not specified explicitly, infer it from parse fn.
  try {
    return coderRegistry.getCoder(TypeDescriptors.outputOf(parseFn));
  } catch (CannotProvideCoderException e) {
    throw new IllegalArgumentException(
        "Unable to infer coder for output of parseFn. Specify it explicitly using withCoder().",
        e);
  }
}
 
Example #9
Source File: FileBasedSink.java    From beam with Apache License 2.0 6 votes vote down vote up
final Coder<DestinationT> getDestinationCoderWithDefault(CoderRegistry registry)
    throws CannotProvideCoderException {
  Coder<DestinationT> destinationCoder = getDestinationCoder();
  if (destinationCoder != null) {
    return destinationCoder;
  }
  // If dynamicDestinations doesn't provide a coder, try to find it in the coder registry.
  @Nullable
  TypeDescriptor<DestinationT> descriptor =
      extractFromTypeParameters(
          this,
          DynamicDestinations.class,
          new TypeVariableExtractor<
              DynamicDestinations<UserT, DestinationT, OutputT>, DestinationT>() {});
  try {
    return registry.getCoder(descriptor);
  } catch (CannotProvideCoderException e) {
    throw new CannotProvideCoderException(
        "Failed to infer coder for DestinationT from type "
            + descriptor
            + ", please provide it explicitly by overriding getDestinationCoder()",
        e);
  }
}
 
Example #10
Source File: ProtoCoder.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public <T> Coder<T> coderFor(
    TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
    throws CannotProvideCoderException {
  if (!typeDescriptor.isSubtypeOf(MESSAGE_TYPE)) {
    throw new CannotProvideCoderException(
        String.format(
            "Cannot provide %s because %s is not a subclass of %s",
            ProtoCoder.class.getSimpleName(), typeDescriptor, Message.class.getName()));
  }

  @SuppressWarnings("unchecked")
  TypeDescriptor<? extends Message> messageType =
      (TypeDescriptor<? extends Message>) typeDescriptor;
  try {
    @SuppressWarnings("unchecked")
    Coder<T> coder = (Coder<T>) ProtoCoder.of(messageType);
    return coder;
  } catch (IllegalArgumentException e) {
    throw new CannotProvideCoderException(e);
  }
}
 
Example #11
Source File: CombineTranslation.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<KV<K, InputT>> mainInput =
        (PCollection<KV<K, InputT>>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(),
        ((KvCoder<K, InputT>) mainInput.getCoder()).getValueCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 
Example #12
Source File: AppliedCombineFn.java    From beam with Apache License 2.0 6 votes vote down vote up
public static <K, InputT, AccumT, OutputT>
    AppliedCombineFn<K, InputT, AccumT, OutputT> withInputCoder(
        GlobalCombineFn<? super InputT, AccumT, OutputT> fn,
        CoderRegistry registry,
        KvCoder<K, InputT> kvCoder,
        Iterable<PCollectionView<?>> sideInputViews,
        WindowingStrategy<?, ?> windowingStrategy) {
  // Casting down the K and InputT is safe because they're only used as inputs.
  @SuppressWarnings("unchecked")
  GlobalCombineFn<InputT, AccumT, OutputT> clonedFn =
      (GlobalCombineFn<InputT, AccumT, OutputT>) SerializableUtils.clone(fn);
  try {
    Coder<AccumT> accumulatorCoder =
        clonedFn.getAccumulatorCoder(registry, kvCoder.getValueCoder());
    return create(clonedFn, accumulatorCoder, sideInputViews, kvCoder, windowingStrategy);
  } catch (CannotProvideCoderException e) {
    throw new IllegalStateException("Could not determine coder for accumulator", e);
  }
}
 
Example #13
Source File: CombineTranslation.java    From beam with Apache License 2.0 6 votes vote down vote up
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<
            PCollection<KV<K, Iterable<InputT>>>, ?, Combine.GroupedValues<K, InputT, ?>>
        transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<KV<K, Iterable<InputT>>> mainInput =
        (PCollection<KV<K, Iterable<InputT>>>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    KvCoder<K, Iterable<InputT>> kvCoder = (KvCoder<K, Iterable<InputT>>) mainInput.getCoder();
    IterableCoder<InputT> iterCoder = (IterableCoder<InputT>) kvCoder.getValueCoder();
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(), iterCoder.getElemCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 
Example #14
Source File: StateSpecs.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * <b><i>For internal use only; no backwards-compatibility guarantees.</i></b>
 *
 * <p>Create a state spec for values that use a {@link CombineFn} to automatically merge multiple
 * {@code InputT}s into a single {@code OutputT}.
 *
 * <p>This determines the {@code Coder<AccumT>} from the given {@code Coder<InputT>}, and should
 * only be used to initialize static values.
 */
@Internal
public static <InputT, AccumT, OutputT>
    StateSpec<CombiningState<InputT, AccumT, OutputT>> combiningFromInputInternal(
        Coder<InputT> inputCoder, CombineFn<InputT, AccumT, OutputT> combineFn) {
  try {
    Coder<AccumT> accumCoder = combineFn.getAccumulatorCoder(STANDARD_REGISTRY, inputCoder);
    return combiningInternal(accumCoder, combineFn);
  } catch (CannotProvideCoderException e) {
    throw new IllegalArgumentException(
        "Unable to determine accumulator coder for "
            + combineFn.getClass().getSimpleName()
            + " from "
            + inputCoder,
        e);
  }
}
 
Example #15
Source File: CombineFns.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public Coder<Object[]> getAccumulatorCoder(CoderRegistry registry, Coder<DataT> dataCoder)
    throws CannotProvideCoderException {
  List<Coder<Object>> coders = Lists.newArrayList();
  for (int i = 0; i < combineFnCount; ++i) {
    Coder<Object> inputCoder =
        combineInputCoders.get(i).isPresent()
            ? combineInputCoders.get(i).get()
            : registry.getOutputCoder(extractInputFns.get(i), dataCoder);
    coders.add(combineFns.get(i).getAccumulatorCoder(registry, inputCoder));
  }
  return new ComposedAccumulatorCoder(coders);
}
 
Example #16
Source File: ParDo.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PCollection<OutputT> expand(PCollection<? extends InputT> input) {
  SchemaRegistry schemaRegistry = input.getPipeline().getSchemaRegistry();
  CoderRegistry coderRegistry = input.getPipeline().getCoderRegistry();
  finishSpecifyingStateSpecs(fn, coderRegistry, schemaRegistry, input.getCoder());
  TupleTag<OutputT> mainOutput = new TupleTag<>(MAIN_OUTPUT_TAG);
  PCollection<OutputT> res =
      input.apply(withOutputTags(mainOutput, TupleTagList.empty())).get(mainOutput);

  TypeDescriptor<OutputT> outputTypeDescriptor = getFn().getOutputTypeDescriptor();
  try {
    res.setSchema(
        schemaRegistry.getSchema(outputTypeDescriptor),
        outputTypeDescriptor,
        schemaRegistry.getToRowFunction(outputTypeDescriptor),
        schemaRegistry.getFromRowFunction(outputTypeDescriptor));
  } catch (NoSuchSchemaException e) {
    try {
      res.setCoder(
          coderRegistry.getCoder(
              outputTypeDescriptor,
              getFn().getInputTypeDescriptor(),
              ((PCollection<InputT>) input).getCoder()));
    } catch (CannotProvideCoderException e2) {
      // Ignore and leave coder unset.
    }
  }

  return res;
}
 
Example #17
Source File: FilterTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is
 * not useful. If this test ever fails there may be simplifications available to us.
 */
@Test
public void testFilterParDoOutputTypeDescriptorRawWithLambda() throws Exception {

  @SuppressWarnings({"unchecked", "rawtypes"})
  PCollection<String> output = p.apply(Create.of("hello")).apply(Filter.by(s -> true));

  thrown.expect(CannotProvideCoderException.class);
  p.getCoderRegistry().getCoder(output.getTypeDescriptor());
}
 
Example #18
Source File: Latest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public Coder<T> getDefaultOutputCoder(
    CoderRegistry registry, Coder<TimestampedValue<T>> inputCoder)
    throws CannotProvideCoderException {
  checkState(
      inputCoder instanceof TimestampedValue.TimestampedValueCoder,
      "inputCoder must be a TimestampedValueCoder, but was %s",
      inputCoder);

  TimestampedValue.TimestampedValueCoder<T> inputTVCoder =
      (TimestampedValue.TimestampedValueCoder<T>) inputCoder;
  return NullableCoder.of(inputTVCoder.getValueCoder());
}
 
Example #19
Source File: LazyAvroCoder.java    From components with Apache License 2.0 5 votes vote down vote up
@Override
public <T> Coder<T> coderFor(TypeDescriptor<T> typeDescriptor, List<? extends Coder<?>> componentCoders)
        throws CannotProvideCoderException {

    Type t = typeDescriptor.getType();
    if (IndexedRecord.class.isAssignableFrom(typeDescriptor.getRawType())) {
        Coder<T> c = LazyAvroCoder.<T> of();
        return c;
    }
    throw new CannotProvideCoderException(String.format("Cannot provide %s because %s is not implement IndexedRecord",
            LazyAvroCoder.class.getSimpleName(), typeDescriptor));
}
 
Example #20
Source File: PartitionTest.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is
 * not useful. If this test ever fails there may be simplifications available to us.
 */
@Test
@Category(NeedsRunner.class)
public void testPartitionFnOutputTypeDescriptorRaw() throws Exception {

  PCollectionList<String> output =
      pipeline.apply(Create.of("hello")).apply(Partition.of(1, (element, numPartitions) -> 0));

  thrown.expect(CannotProvideCoderException.class);
  pipeline.getCoderRegistry().getCoder(output.get(0).getTypeDescriptor());
}
 
Example #21
Source File: BigQueryIO.java    From beam with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
Coder<T> inferCoder(CoderRegistry coderRegistry) {
  if (getCoder() != null) {
    return getCoder();
  }

  try {
    return coderRegistry.getCoder(TypeDescriptors.outputOf(getParseFn()));
  } catch (CannotProvideCoderException e) {
    throw new IllegalArgumentException(
        "Unable to infer coder for output of parseFn. Specify it explicitly using withCoder().",
        e);
  }
}
 
Example #22
Source File: DynamicDestinationsHelpers.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
Coder<DestinationT> getDestinationCoderWithDefault(CoderRegistry registry)
    throws CannotProvideCoderException {
  Coder<DestinationT> destinationCoder = getDestinationCoder();
  if (destinationCoder != null) {
    return destinationCoder;
  }
  return inner.getDestinationCoderWithDefault(registry);
}
 
Example #23
Source File: HadoopFormatIO.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the default coder for a given type descriptor. Coder Registry is queried for correct
 * coder, if not found in Coder Registry, then check if the type descriptor provided is of type
 * Writable, then WritableCoder is returned, else exception is thrown "Cannot find coder".
 */
@SuppressWarnings({"unchecked", "WeakerAccess"})
public <T> Coder<T> getDefaultCoder(TypeDescriptor<?> typeDesc, CoderRegistry coderRegistry) {
  Class classType = typeDesc.getRawType();
  try {
    return (Coder<T>) coderRegistry.getCoder(typeDesc);
  } catch (CannotProvideCoderException e) {
    if (Writable.class.isAssignableFrom(classType)) {
      return (Coder<T>) WritableCoder.of(classType);
    }
    throw new IllegalStateException(
        String.format("Cannot find coder for %s  : ", typeDesc) + e.getMessage(), e);
  }
}
 
Example #24
Source File: KuduIO.java    From beam with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
Coder<T> inferCoder(CoderRegistry coderRegistry) {
  try {
    return getCoder() != null
        ? getCoder()
        : coderRegistry.getCoder(TypeDescriptors.outputOf(getParseFn()));
  } catch (CannotProvideCoderException e) {
    throw new IllegalArgumentException(
        "Unable to infer coder for output of parseFn ("
            + TypeDescriptors.outputOf(getParseFn())
            + "). Specify it explicitly using withCoder().",
        e);
  }
}
 
Example #25
Source File: LocalDeserializerProvider.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Attempt to infer a {@link Coder} by extracting the type of the deserialized-class from the
 * deserializer argument using the {@link Coder} registry.
 */
@Override
public NullableCoder<T> getCoder(CoderRegistry coderRegistry) {
  for (Type type : deserializer.getGenericInterfaces()) {
    if (!(type instanceof ParameterizedType)) {
      continue;
    }

    // This does not recurse: we will not infer from a class that extends
    // a class that extends Deserializer<T>.
    ParameterizedType parameterizedType = (ParameterizedType) type;

    if (parameterizedType.getRawType() == Deserializer.class) {
      Type parameter = parameterizedType.getActualTypeArguments()[0];

      @SuppressWarnings("unchecked")
      Class<T> clazz = (Class<T>) parameter;

      try {
        return NullableCoder.of(coderRegistry.getCoder(clazz));
      } catch (CannotProvideCoderException e) {
        throw new RuntimeException(
            String.format(
                "Unable to automatically infer a Coder for "
                    + "the Kafka Deserializer %s: no coder registered for type %s",
                deserializer, clazz));
      }
    }
  }
  throw new RuntimeException(
      String.format("Could not extract the Kafka Deserializer type from %s", deserializer));
}
 
Example #26
Source File: CombineTranslation.java    From beam with Apache License 2.0 5 votes vote down vote up
private static <InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<PCollection<InputT>, ?, Combine.Globally<InputT, ?>> transform)
    throws IOException {
  try {
    @SuppressWarnings("unchecked")
    PCollection<InputT> mainInput =
        (PCollection<InputT>)
            Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
    return combineFn.getAccumulatorCoder(
        transform.getPipeline().getCoderRegistry(), mainInput.getCoder());
  } catch (CannotProvideCoderException e) {
    throw new IOException("Could not obtain a Coder for the accumulator", e);
  }
}
 
Example #27
Source File: ForwardingPTransform.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public OutputT expand(InputT input) {
  OutputT res = delegate().expand(input);
  if (res instanceof PCollection) {
    PCollection pc = (PCollection) res;
    try {
      pc.setCoder(delegate().getDefaultOutputCoder(input, pc));
    } catch (CannotProvideCoderException e) {
      // Let coder inference happen later.
    }
  }
  return res;
}
 
Example #28
Source File: MultiStepCombine.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, InputT>> input) {
  checkArgument(
      input.getCoder() instanceof KvCoder,
      "Expected input to have a %s of type %s, got %s",
      Coder.class.getSimpleName(),
      KvCoder.class.getSimpleName(),
      input.getCoder());
  KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) input.getCoder();
  Coder<InputT> inputValueCoder = inputCoder.getValueCoder();
  Coder<AccumT> accumulatorCoder;
  try {
    accumulatorCoder =
        combineFn.getAccumulatorCoder(input.getPipeline().getCoderRegistry(), inputValueCoder);
  } catch (CannotProvideCoderException e) {
    throw new IllegalStateException(
        String.format(
            "Could not construct an Accumulator Coder with the provided %s %s",
            CombineFn.class.getSimpleName(), combineFn),
        e);
  }
  return input
      .apply(
          ParDo.of(
              new CombineInputs<>(
                  combineFn,
                  input.getWindowingStrategy().getTimestampCombiner(),
                  inputCoder.getKeyCoder())))
      .setCoder(KvCoder.of(inputCoder.getKeyCoder(), accumulatorCoder))
      .apply(GroupByKey.create())
      .apply(new MergeAndExtractAccumulatorOutput<>(combineFn, outputCoder));
}
 
Example #29
Source File: CopyOnAccessInMemoryStateInternalsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException {
  CopyOnAccessInMemoryStateInternals<String> underlying =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs();

  StateNamespace namespace = new StateNamespaceForTest("foo");
  CoderRegistry reg = pipeline.getCoderRegistry();
  StateTag<CombiningState<Long, long[], Long>> stateTag =
      StateTags.combiningValue(
          "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn);
  GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag);
  assertThat(underlyingValue.read(), equalTo(0L));

  underlyingValue.add(1L);
  assertThat(underlyingValue.read(), equalTo(1L));

  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);
  GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag);
  assertThat(copyOnAccessState.read(), equalTo(1L));

  copyOnAccessState.add(4L);
  assertThat(copyOnAccessState.read(), equalTo(5L));
  assertThat(underlyingValue.read(), equalTo(1L));

  GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag);
  assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read()));
}
 
Example #30
Source File: KafkaIO.java    From DataflowTemplates with Apache License 2.0 5 votes vote down vote up
/**
 * Attempt to infer a {@link Coder} by extracting the type of the deserialized-class from the
 * deserializer argument using the {@link Coder} registry.
 */
@VisibleForTesting
static <T> NullableCoder<T> inferCoder(
    CoderRegistry coderRegistry, Class<? extends Deserializer<T>> deserializer) {
  checkNotNull(deserializer);

  for (Type type : deserializer.getGenericInterfaces()) {
    if (!(type instanceof ParameterizedType)) {
      continue;
    }

    // This does not recurse: we will not infer from a class that extends
    // a class that extends Deserializer<T>.
    ParameterizedType parameterizedType = (ParameterizedType) type;

    if (parameterizedType.getRawType() == Deserializer.class) {
      Type parameter = parameterizedType.getActualTypeArguments()[0];

      @SuppressWarnings("unchecked")
      Class<T> clazz = (Class<T>) parameter;

      try {
        return NullableCoder.of(coderRegistry.getCoder(clazz));
      } catch (CannotProvideCoderException e) {
        throw new RuntimeException(
            String.format(
                "Unable to automatically infer a Coder for "
                    + "the Kafka Deserializer %s: no coder registered for type %s",
                deserializer, clazz));
      }
    }
  }

  throw new RuntimeException(
      String.format("Could not extract the Kafka Deserializer type from %s", deserializer));
}