Java Code Examples for org.apache.beam.sdk.coders.CoderException

The following examples show how to use org.apache.beam.sdk.coders.CoderException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kettle-beam   Source File: KettleRowCoder.java    License: Apache License 2.0 8 votes vote down vote up
@Override public KettleRow decode( InputStream inStream ) throws CoderException, IOException {

    ObjectInputStream in = new ObjectInputStream( inStream );

    Object[] row = null;
    int length = in.readInt();
    if ( length < 0 ) {
      return new KettleRow( row );
    }
    row = new Object[ length ];
    for ( int i = 0; i < length; i++ ) {
      // Null?
      boolean isNull = in.readBoolean();
      if ( !isNull ) {
        int objectType = in.readInt();
        Object object = read( in, objectType );
        row[i] = object;
      }
    }

    return new KettleRow(row);
  }
 
Example 2
Source Project: beam   Source File: MapToTupleFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Tuple<byte[], byte[]> map(WindowedValue<KV<K, V>> input) {
  Tuple<byte[], byte[]> element = null;

  WindowedValue<KV<K, WindowedValue<V>>> temp =
      WindowedValue.of(
          KV.of(
              input.getValue().getKey(),
              WindowedValue.of(
                  input.getValue().getValue(),
                  input.getTimestamp(),
                  input.getWindows(),
                  input.getPane())),
          input.getTimestamp(),
          input.getWindows(),
          input.getPane());
  try {
    element =
        new Tuple<>(
            CoderUtils.encodeToByteArray(keyCoder, temp.getValue().getKey()),
            CoderUtils.encodeToByteArray(wvCoder, temp.getValue().getValue()));
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return element;
}
 
Example 3
Source Project: twister2   Source File: ByteToWindowFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public KV<K, Iterable<WindowedValue<V>>> map(Tuple<byte[], Iterator<byte[]>> input) {
  K key = null;
  Iterable<WindowedValue<V>> value = null;
  try {
    key = CoderUtils.decodeFromByteArray(keyCoder, input.getKey());

    value = StreamSupport.stream(
        Spliterators.spliteratorUnknownSize(input.getValue(), Spliterator.ORDERED), false)
        .map(bytes -> TranslationUtils.fromByteArray(bytes, wvCoder))
        .collect(Collectors.toList());
  } catch (CoderException e) {
    e.printStackTrace();
  }
  return KV.of(key, value);
}
 
Example 4
Source Project: beam   Source File: ByteToWindowFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public KV<K, Iterable<WindowedValue<V>>> map(Tuple<byte[], Iterator<byte[]>> input) {
  K key = null;
  Iterable<WindowedValue<V>> value = null;
  try {
    key = CoderUtils.decodeFromByteArray(keyCoder, input.getKey());
    // TODO need to replace this with a custom iterator
    value =
        StreamSupport.stream(
                Spliterators.spliteratorUnknownSize(input.getValue(), Spliterator.ORDERED), false)
            .map(bytes -> TranslationUtils.fromByteArray(bytes, wvCoder))
            .collect(Collectors.toList());
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return KV.of(key, value);
}
 
Example 5
Source Project: beam   Source File: PAssert.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PCollectionView<ActualT> expand(PBegin input) {
  final Coder<T> coder = actual.getCoder();
  return actual
      .apply("FilterActuals", rewindowActuals.prepareActuals())
      .apply("GatherPanes", GatherAllPanes.globally())
      .apply("ExtractPane", MapElements.via(extractPane))
      .setCoder(IterableCoder.of(actual.getCoder()))
      .apply(Flatten.iterables())
      .apply("RewindowActuals", rewindowActuals.windowActuals())
      .apply(
          ParDo.of(
              new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext context) throws CoderException {
                  context.output(CoderUtils.clone(coder, context.element()));
                }
              }))
      .apply(actualView);
}
 
Example 6
Source Project: beam   Source File: CoderProperties.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static <T> T decode(Coder<T> coder, Coder.Context context, byte[] bytes)
    throws CoderException, IOException {
  @SuppressWarnings("unchecked")
  Coder<T> deserializedCoder = SerializableUtils.clone(coder);

  byte[] buffer;
  if (Objects.equals(context, Coder.Context.NESTED)) {
    buffer = new byte[bytes.length + 1];
    System.arraycopy(bytes, 0, buffer, 0, bytes.length);
    buffer[bytes.length] = 1;
  } else {
    buffer = bytes;
  }

  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(buffer));
  T value = deserializedCoder.decode(new UnownedInputStream(cis), context);
  assertThat(
      "consumed bytes equal to encoded bytes", cis.getCount(), equalTo((long) bytes.length));
  return value;
}
 
Example 7
Source Project: beam   Source File: MutationDetectors.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyUnmodifiedThrowingCheckedExceptions() throws CoderException {
  // Since there is no guarantee that cloning an object via the coder will
  // return the exact same type as value, We are cloning the possiblyModifiedObject
  // before getting it's structural value. This way we are guaranteed to compare the same
  // types.
  T possiblyModifiedClonedValue = CoderUtils.clone(coder, possiblyModifiedObject);
  Object newStructuralValue = coder.structuralValue(possiblyModifiedClonedValue);
  if (originalStructuralValue.equals(newStructuralValue)) {
    return;
  } else if (Objects.deepEquals(
      encodedOriginalObject, CoderUtils.encodeToByteArray(coder, possiblyModifiedObject))) {
    LOG.warn(
        "{} of type {} has a #structuralValue method which does not return true when the "
            + "encoding of the elements is equal. Element {}",
        Coder.class.getSimpleName(),
        coder.getClass(),
        possiblyModifiedObject);
    return;
  }
  illegalMutation(clonedOriginalObject, possiblyModifiedClonedValue);
}
 
Example 8
Source Project: hop   Source File: HopRowCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override public void encode( HopRow value, OutputStream outStream ) throws CoderException, IOException {

    Object[] row = value.getRow();
    ObjectOutputStream out = new ObjectOutputStream( outStream );

    // Length
    //
    if ( row == null ) {
      out.writeInt( -1 );
      return; // all done
    } else {
      out.writeInt( row.length );
    }

    // The values
    //
    for ( int i = 0; i < row.length; i++ ) {
      Object object = row[ i ];
      // Null?
      //
      out.writeBoolean( object == null );

      if ( object != null ) {
        // Type?
        //
        int objectType = getObjectType( object );
        out.writeInt( objectType );

        // The object itself
        //
        write( out, objectType, object );
      }
    }
    out.flush();
  }
 
Example 9
Source Project: beam   Source File: AwsCoders.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SdkHttpMetadata decode(InputStream inStream) throws CoderException, IOException {
  final int httpStatusCode = STATUS_CODE_CODER.decode(inStream);
  HttpResponse httpResponse = new HttpResponse(null, null);
  httpResponse.setStatusCode(httpStatusCode);
  if (includeHeaders) {
    Optional.ofNullable(HEADERS_ENCODER.decode(inStream))
        .ifPresent(
            headers ->
                headers.keySet().forEach(k -> httpResponse.addHeader(k, headers.get(k))));
  }
  return SdkHttpMetadata.from(httpResponse);
}
 
Example 10
Source Project: incubator-nemo   Source File: BeamDecoderFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Decode the actual data internally.
 *
 * @return the decoded data.
 * @throws IOException if fail to decode.
 */
protected T2 decodeInternal() throws IOException {
  try {
    return beamCoder.decode(inputStream);
  } catch (final CoderException e) {
    throw new IOException(e);
  }
}
 
Example 11
Source Project: beam   Source File: StreamingSideInputFetcher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(Windmill.GlobalDataRequest value, OutputStream outStream, Context context)
    throws IOException {
  if (value == null) {
    throw new CoderException("cannot encode a null " + protoMessageClass.getSimpleName());
  }
  if (context.isWholeStream) {
    value.writeTo(outStream);
  } else {
    value.writeDelimitedTo(outStream);
  }
}
 
Example 12
Source Project: beam   Source File: WindowedValue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public WindowedValue<T> decode(InputStream inStream, Context context)
    throws CoderException, IOException {
  Instant timestamp = InstantCoder.of().decode(inStream);
  Collection<? extends BoundedWindow> windows = windowsCoder.decode(inStream);
  PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream);
  T value = valueCoder.decode(inStream, context);

  // Because there are some remaining (incorrect) uses of WindowedValue with no windows,
  // we call this deprecated no-validation path when decoding
  return WindowedValue.createWithoutValidation(value, timestamp, windows, pane);
}
 
Example 13
Source Project: DataflowTemplates   Source File: FailsafeElementCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(FailsafeElement<OriginalT, CurrentT> value, OutputStream outStream)
    throws IOException {
  if (value == null) {
    throw new CoderException("The FailsafeElementCoder cannot encode a null object!");
  }

  originalPayloadCoder.encode(value.getOriginalPayload(), outStream);
  currentPayloadCoder.encode(value.getPayload(), outStream);
  STRING_CODER.encode(value.getErrorMessage(), outStream);
  STRING_CODER.encode(value.getStacktrace(), outStream);
}
 
Example 14
Source Project: beam   Source File: FileBasedSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(FileResult<DestinationT> value, OutputStream outStream) throws IOException {
  if (value == null) {
    throw new CoderException("cannot encode a null value");
  }
  FILENAME_CODER.encode(value.getTempFilename().toString(), outStream);
  windowCoder.encode(value.getWindow(), outStream);
  PANE_INFO_CODER.encode(value.getPaneInfo(), outStream);
  SHARD_CODER.encode(value.getShard(), outStream);
  destinationCoder.encode(value.getDestination(), outStream);
}
 
Example 15
Source Project: nemo   Source File: BeamCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(final T value, final OutputStream outStream) throws IOException {
  if (beamCoder instanceof VoidCoder) {
    outStream.write(0);
    return;
  }
  try {
    beamCoder.encode(value, outStream);
  } catch (final CoderException e) {
    throw new IOException(e);
  }
}
 
Example 16
Source Project: nemo   Source File: BeamCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public T decode(final InputStream inStream) throws IOException {
  if (beamCoder instanceof VoidCoder && inStream.read() == -1) {
    throw new IOException("End of stream reached");
  }
  try {
    return beamCoder.decode(inStream);
  } catch (final CoderException e) {
    throw new IOException(e);
  }
}
 
Example 17
Source Project: nemo   Source File: PairCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(final Pair<A, B> pair, final OutputStream outStream) throws IOException {
  if (pair == null) {
    throw new CoderException("cannot encode a null KV");
  }
  leftCoder.encode(pair.left(), outStream);
  rightCoder.encode(pair.right(), outStream);
}
 
Example 18
Source Project: beam   Source File: CoderUtils.java    License: Apache License 2.0 5 votes vote down vote up
/** Parses a value from a base64-encoded String using the given coder. */
public static <T> T decodeFromBase64(Coder<T> coder, String encodedValue) throws CoderException {
  return decodeFromSafeStream(
      coder,
      new ByteArrayInputStream(BaseEncoding.base64Url().omitPadding().decode(encodedValue)),
      Coder.Context.OUTER);
}
 
Example 19
Source Project: beam   Source File: DirectRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnencodableOutputFromUnboundedRead() {
  Pipeline p = getPipeline();
  p.apply(GenerateSequence.from(0)).setCoder(new LongNoDecodeCoder());

  thrown.expectCause(isA(CoderException.class));
  thrown.expectMessage("Cannot decode a long");
  p.run();
}
 
Example 20
Source Project: beam   Source File: InMemoryReader.java    License: Apache License 2.0 5 votes vote down vote up
private T decode(String encodedElementString) throws CoderException {
  // TODO: Replace with the real encoding used by the
  // front end, when we know what it is.
  byte[] encodedElement = StringUtils.jsonStringToByteArray(encodedElementString);
  notifyElementRead(encodedElement.length);
  return CoderUtils.decodeFromByteArray(coder, encodedElement);
}
 
Example 21
Source Project: beam   Source File: SerializableMatchers.java    License: Apache License 2.0 5 votes vote down vote up
public SerializableArrayViaCoder(Coder<T> elementCoder, T[] value) {
  this.coder = ListCoder.of(elementCoder);
  this.value = value;
  try {
    this.encodedValue = CoderUtils.encodeToByteArray(coder, Arrays.asList(value));
  } catch (CoderException exc) {
    throw UserCodeException.wrap(exc);
  }
}
 
Example 22
Source Project: beam   Source File: CoderUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCoderExceptionPropagation() throws Exception {
  @SuppressWarnings("unchecked")
  Coder<String> crashingCoder = mock(Coder.class);
  doThrow(new CoderException("testing exception"))
      .when(crashingCoder)
      .encode(anyString(), any(OutputStream.class), any(Coder.Context.class));

  expectedException.expect(CoderException.class);
  expectedException.expectMessage("testing exception");

  CoderUtils.encodeToByteArray(crashingCoder, "hello");
}
 
Example 23
Source Project: beam   Source File: MutationDetectors.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void verifyUnmodified() {
  try {
    verifyUnmodifiedThrowingCheckedExceptions();
  } catch (CoderException exn) {
    throw new RuntimeException(exn);
  }
}
 
Example 24
Source Project: beam   Source File: ApproximateDistinct.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected long getEncodedElementByteSize(HyperLogLogPlus value) throws IOException {
  if (value == null) {
    throw new CoderException("cannot encode a null HyperLogLogPlus sketch");
  }
  return value.sizeof();
}
 
Example 25
Source Project: beam   Source File: AttributeValueCoder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(AttributeValue value, OutputStream outStream) throws IOException {

  if (value.getS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.s.toString(), outStream);
    StringUtf8Coder.of().encode(value.getS(), outStream);
  } else if (value.getN() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.n.toString(), outStream);
    StringUtf8Coder.of().encode(value.getN(), outStream);
  } else if (value.getBOOL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.bOOL.toString(), outStream);
    BooleanCoder.of().encode(value.getBOOL(), outStream);
  } else if (value.getB() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.b.toString(), outStream);
    ByteArrayCoder.of().encode(convertToByteArray(value.getB()), outStream);
  } else if (value.getSS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.sS.toString(), outStream);
    LIST_STRING_CODER.encode(value.getSS(), outStream);
  } else if (value.getNS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.nS.toString(), outStream);
    LIST_STRING_CODER.encode(value.getNS(), outStream);
  } else if (value.getBS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.bS.toString(), outStream);
    LIST_BYTE_CODER.encode(convertToListByteArray(value.getBS()), outStream);
  } else if (value.getL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.l.toString(), outStream);
    LIST_ATTRIBUTE_CODER.encode(value.getL(), outStream);
  } else if (value.getM() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.m.toString(), outStream);
    MAP_ATTRIBUTE_CODER.encode(value.getM(), outStream);
  } else if (value.getNULL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.nULLValue.toString(), outStream);
    BooleanCoder.of().encode(value.getNULL(), outStream);
  } else {
    throw new CoderException("Unknown Type");
  }
}
 
Example 26
Source Project: beam   Source File: KryoCoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = CoderException.class)
public void testWrongRegistrarDecoding() throws IOException {
  final KryoRegistrar registrarCoding = k -> k.register(ClassToBeEncoded.class);
  final KryoRegistrar registrarDecoding =
      k -> {
        // No-op
      };
  final KryoCoder<ClassToBeEncoded> coderToEncode = KryoCoder.of(OPTIONS, registrarCoding);
  final KryoCoder<ClassToBeEncoded> coderToDecode = KryoCoder.of(OPTIONS, registrarDecoding);
  assertEncoding(coderToEncode, coderToDecode);
}
 
Example 27
Source Project: beam   Source File: TimerInternals.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(TimerData timer, OutputStream outStream) throws CoderException, IOException {
  STRING_CODER.encode(timer.getTimerId(), outStream);
  STRING_CODER.encode(timer.getNamespace().stringKey(), outStream);
  INSTANT_CODER.encode(timer.getTimestamp(), outStream);
  STRING_CODER.encode(timer.getDomain().name(), outStream);
}
 
Example 28
Source Project: beam   Source File: ProtoCoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void encodeNullThrowsCoderException() throws Exception {
  thrown.expect(CoderException.class);
  thrown.expectMessage("cannot encode a null MessageA");

  CoderUtils.encodeToBase64(ProtoCoder.of(MessageA.class), null);
}
 
Example 29
Source Project: beam   Source File: TimerInternals.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TimerData decode(InputStream inStream) throws CoderException, IOException {
  String timerId = STRING_CODER.decode(inStream);
  StateNamespace namespace =
      StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder);
  Instant timestamp = INSTANT_CODER.decode(inStream);
  TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream));
  return TimerData.of(timerId, namespace, timestamp, timestamp, domain);
}
 
Example 30
Source Project: beam   Source File: CloningBundleFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Record decode(InputStream inStream) throws CoderException, IOException {
  return new Record() {
    @Override
    public String toString() {
      return "DecodedRecord";
    }
  };
}