Java Code Examples for org.apache.avro.io.Encoder

The following examples show how to use org.apache.avro.io.Encoder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: avro-fastserde   Source File: FastSerializerGenerator.java    License: Apache License 2.0 6 votes vote down vote up
private JMethod createMethod(final Schema schema) {
    if (Schema.Type.RECORD.equals(schema.getType())) {
        if (!methodAlreadyDefined(schema)) {
            JMethod method = serializerClass.method(JMod.PUBLIC, codeModel.VOID,
                    "serialize" + schema.getName() + nextRandomInt());
            method._throws(IOException.class);
            method.param(schemaAssistant.classFromSchema(schema), "data");
            method.param(Encoder.class, ENCODER);

            method.annotate(SuppressWarnings.class).param("value", "unchecked");
            serializeMethodMap.put(schema.getFullName(), method);

            return method;
        } else {
            throw new FastSerializerGeneratorException("Method already exists for: " + schema.getFullName());
        }
    }
    throw new FastSerializerGeneratorException("No method for schema type: " + schema.getType());
}
 
Example 2
Source Project: beam   Source File: BigQueryIOStorageQueryTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ReadRowsResponse createResponse(
    Schema schema, Collection<GenericRecord> genericRecords, double fractionConsumed)
    throws Exception {
  GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  Encoder binaryEncoder = ENCODER_FACTORY.binaryEncoder(outputStream, null);
  for (GenericRecord genericRecord : genericRecords) {
    writer.write(genericRecord, binaryEncoder);
  }

  binaryEncoder.flush();

  return ReadRowsResponse.newBuilder()
      .setAvroRows(
          AvroRows.newBuilder()
              .setSerializedBinaryRows(ByteString.copyFrom(outputStream.toByteArray()))
              .setRowCount(genericRecords.size()))
      .setStatus(StreamStatus.newBuilder().setFractionConsumed((float) fractionConsumed))
      .build();
}
 
Example 3
public void serialize(List<Long> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        if (data instanceof PrimitiveLongList) {
            PrimitiveLongList primitiveList0 = null;
            primitiveList0 = ((PrimitiveLongList) data);
            for (int counter0 = 0; (counter0 <primitiveList0 .size()); counter0 ++) {
                (encoder).startItem();
                (encoder).writeLong(primitiveList0 .getPrimitive(counter0));
            }
        } else {
            for (int counter1 = 0; (counter1 <data.size()); counter1 ++) {
                (encoder).startItem();
                (encoder).writeLong(data.get(counter1));
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 4
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 5
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 6
Source Project: iceberg   Source File: ValueWriters.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void write(BigDecimal decimal, Encoder encoder) throws IOException {
  Preconditions.checkArgument(decimal.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, decimal);
  Preconditions.checkArgument(decimal.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, decimal);

  byte fillByte = (byte) (decimal.signum() < 0 ? 0xFF : 0x00);
  byte[] unscaled = decimal.unscaledValue().toByteArray();
  byte[] buf = bytes.get();
  int offset = length - unscaled.length;

  for (int i = 0; i < length; i += 1) {
    if (i < offset) {
      buf[i] = fillByte;
    } else {
      buf[i] = unscaled[i - offset];
    }
  }

  encoder.writeFixed(buf);
}
 
Example 7
@Override
@SuppressWarnings("unchecked")
public void write(T data, Encoder out) throws IOException {
  FastSerializer<T> fastSerializer;
  if (cachedFastSerializer != null) {
    fastSerializer = cachedFastSerializer;
  } else {
    fastSerializer = getFastSerializerFromCache(cache, writerSchema);
    if (fastSerializer instanceof FastSerdeCache.FastSerializerWithAvroSpecificImpl
        || fastSerializer instanceof FastSerdeCache.FastSerializerWithAvroGenericImpl) {
      // don't cache
    } else {
      cachedFastSerializer = fastSerializer;
      if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("FastSerializer has been generated and cached for writer schema: [" + writerSchema + "]");
      }
    }
  }

  fastSerializer.serialize(data, out);
}
 
Example 8
public void serialize(List<IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (int counter0 = 0; (counter0 <data.size()); counter0 ++) {
            (encoder).startItem();
            IndexedRecord record0 = null;
            record0 = ((List<IndexedRecord> ) data).get(counter0);
            serializeRecord0(record0, (encoder));
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 9
public void serialize(List<IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (int counter0 = 0; (counter0 <data.size()); counter0 ++) {
            (encoder).startItem();
            IndexedRecord union0 = null;
            union0 = ((List<IndexedRecord> ) data).get(counter0);
            if (union0 == null) {
                (encoder).writeIndex(0);
                (encoder).writeNull();
            } else {
                if ((union0 instanceof IndexedRecord)&&"com.adpilot.utils.generated.avro.record".equals(((IndexedRecord) union0).getSchema().getFullName())) {
                    (encoder).writeIndex(1);
                    serializeRecord0(((IndexedRecord) union0), (encoder));
                }
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 10
Source Project: Cubert   Source File: PigAvroDatumWriter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Write double. Users can cast long, float and integer to double.
 * 
 */
protected void writeDouble(Object datum, Encoder out) throws IOException {
    double num;
    if (datum instanceof Integer) {
        num = ((Integer) datum).doubleValue();
    } else if (datum instanceof Long) {
        num = ((Long) datum).doubleValue();
    } else if (datum instanceof Float) {
        num = ((Float) datum).doubleValue();
    } else if (datum instanceof Double) {
        num = (Double) datum;
    } else
        throw new IOException("Cannot convert to double:" + datum.getClass());

    out.writeDouble(num);
}
 
Example 11
public void serialize(List<Integer> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        if (data instanceof PrimitiveIntList) {
            PrimitiveIntList primitiveList0 = null;
            primitiveList0 = ((PrimitiveIntList) data);
            for (int counter0 = 0; (counter0 <primitiveList0 .size()); counter0 ++) {
                (encoder).startItem();
                (encoder).writeInt(primitiveList0 .getPrimitive(counter0));
            }
        } else {
            for (int counter1 = 0; (counter1 <data.size()); counter1 ++) {
                (encoder).startItem();
                (encoder).writeInt(data.get(counter1));
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 12
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 13
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 14
Source Project: pulsar   Source File: AvroSerializationSchema.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public byte[] serialize(T t) {
    if (null == t) {
        return null;
    }

    // Writer to serialize Avro record into a byte array.
    DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());
    // Output stream to serialize records into byte array.
    ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
    // Low-level class for serialization of Avro values.
    Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
    arrayOutputStream.reset();
    try {
        writer.write(t, encoder);
        encoder.flush();
    } catch (IOException e) {
        throw new RuntimeException("Error while serializing the record to Avro", e);
    }

    return arrayOutputStream.toByteArray();
}
 
Example 15
public void serialize(List<IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (int counter0 = 0; (counter0 <data.size()); counter0 ++) {
            (encoder).startItem();
            IndexedRecord record0 = null;
            record0 = ((List<IndexedRecord> ) data).get(counter0);
            serializeRecord0(record0, (encoder));
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 16
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 17
public void serialize(List<Double> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        if (data instanceof PrimitiveDoubleList) {
            PrimitiveDoubleList primitiveList0 = null;
            primitiveList0 = ((PrimitiveDoubleList) data);
            for (int counter0 = 0; (counter0 <primitiveList0 .size()); counter0 ++) {
                (encoder).startItem();
                (encoder).writeDouble(primitiveList0 .getPrimitive(counter0));
            }
        } else {
            for (int counter1 = 0; (counter1 <data.size()); counter1 ++) {
                (encoder).startItem();
                (encoder).writeDouble(data.get(counter1));
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 18
public void serialize(List<IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (int counter0 = 0; (counter0 <data.size()); counter0 ++) {
            (encoder).startItem();
            IndexedRecord union0 = null;
            union0 = ((List<IndexedRecord> ) data).get(counter0);
            if (union0 == null) {
                (encoder).writeIndex(0);
                (encoder).writeNull();
            } else {
                if ((union0 instanceof IndexedRecord)&&"com.adpilot.utils.generated.avro.record".equals(((IndexedRecord) union0).getSchema().getFullName())) {
                    (encoder).writeIndex(1);
                    serializeRecord0(((IndexedRecord) union0), (encoder));
                }
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 19
public void serialize(List<Boolean> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        if (data instanceof PrimitiveBooleanList) {
            PrimitiveBooleanList primitiveList0 = null;
            primitiveList0 = ((PrimitiveBooleanList) data);
            for (int counter0 = 0; (counter0 <primitiveList0 .size()); counter0 ++) {
                (encoder).startItem();
                (encoder).writeBoolean(primitiveList0 .getPrimitive(counter0));
            }
        } else {
            for (int counter1 = 0; (counter1 <data.size()); counter1 ++) {
                (encoder).startItem();
                (encoder).writeBoolean(data.get(counter1));
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 20
public void serialize(List<Long> data, Encoder encoder)
    throws IOException
{
    (encoder).writeArrayStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        if (data instanceof PrimitiveLongList) {
            PrimitiveLongList primitiveList0 = null;
            primitiveList0 = ((PrimitiveLongList) data);
            for (int counter0 = 0; (counter0 <primitiveList0 .size()); counter0 ++) {
                (encoder).startItem();
                (encoder).writeLong(primitiveList0 .getPrimitive(counter0));
            }
        } else {
            for (int counter1 = 0; (counter1 <data.size()); counter1 ++) {
                (encoder).startItem();
                (encoder).writeLong(data.get(counter1));
            }
        }
    }
    (encoder).writeArrayEnd();
}
 
Example 21
Source Project: iceberg   Source File: ValueWriters.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void write(Object s, Encoder encoder) throws IOException {
  // use getBytes because it may return the backing byte array if available.
  // otherwise, it copies to a new byte array, which is still cheaper than Avro
  // calling toString, which incurs encoding costs
  if (s instanceof Utf8) {
    encoder.writeString((Utf8) s);
  } else if (s instanceof String) {
    encoder.writeString(new Utf8((String) s));
  } else if (s == null) {
    throw new IllegalArgumentException("Cannot write null to required string column");
  } else {
    throw new IllegalArgumentException(
        "Cannot write unknown string type: " + s.getClass().getName() + ": " + s.toString());
  }
}
 
Example 22
@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 
Example 23
Source Project: components   Source File: DatasetContentWriter.java    License: Apache License 2.0 6 votes vote down vote up
private Consumer<IndexedRecord> getWritingConsumer(Encoder[] encoder) {
    return new Consumer<IndexedRecord>() {

        GenericDatumWriter<IndexedRecord> writer = null;

        @Override
        public void accept(IndexedRecord ir) {
            if (writer == null) {
                writer = new GenericDatumWriter<>(ir.getSchema());
                try {
                    if (json) {
                        encoder[0] = EncoderFactory.get().jsonEncoder(ir.getSchema(), output);
                    } else {
                        encoder[0] = EncoderFactory.get().binaryEncoder(output, null);
                    }
                } catch (IOException ioe) {
                    throw new RuntimeException(ioe);
                }

            }
            writeIndexedRecord(writer, encoder[0], ir);
        }
    };
}
 
Example 24
Source Project: iceberg   Source File: SparkValueWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void write(ArrayData array, Encoder encoder) throws IOException {
  encoder.writeArrayStart();
  int numElements = array.numElements();
  encoder.setItemCount(numElements);
  for (int i = 0; i < numElements; i += 1) {
    encoder.startItem();
    elementWriter.write((T) array.get(i, elementType), encoder);
  }
  encoder.writeArrayEnd();
}
 
Example 25
Source Project: secor   Source File: AvroSerializer.java    License: Apache License 2.0 5 votes vote down vote up
public static byte[] serialize(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
    writer.write(record, encoder);
    encoder.flush();
    ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length);
    serialized.put(out.toByteArray());
    return serialized.array();
}
 
Example 26
Source Project: iceberg   Source File: SparkValueWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(UTF8String s, Encoder encoder) throws IOException {
  // use getBytes because it may return the backing byte array if available.
  // otherwise, it copies to a new byte array, which is still cheaper than Avro
  // calling toString, which incurs encoding costs
  encoder.writeString(new Utf8(s.getBytes()));
}
 
Example 27
Source Project: iceberg   Source File: SparkValueWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(UTF8String s, Encoder encoder) throws IOException {
  // TODO: direct conversion from string to byte buffer
  UUID uuid = UUID.fromString(s.toString());
  ByteBuffer buffer = BUFFER.get();
  buffer.rewind();
  buffer.putLong(uuid.getMostSignificantBits());
  buffer.putLong(uuid.getLeastSignificantBits());
  encoder.writeFixed(buffer.array());
}
 
Example 28
Source Project: spork   Source File: PigAvroDatumWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called to write a fixed value. 
 */
@Override
protected void writeFixed(Schema schema, Object datum,
                                org.apache.avro.io.Encoder out)
                                throws IOException {
    if (datum instanceof DataByteArray) {
        final byte[] bytes = ((DataByteArray) datum).get();
        out.writeFixed(bytes, 0, bytes.length);
    } else
        throw new RuntimeException("Unsupported type fixed:" + datum.getClass());

}
 
Example 29
Source Project: spork   Source File: PigAvroDatumWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write long. Users can cast integer into long.
 * 
 */
protected void writeLong(Object datum, Encoder out) throws IOException {
    long num;
    if (datum instanceof Integer) {
        num = ((Integer) datum).longValue();
    } else if (datum instanceof Long) {
        num = (Long) datum;
    } else
        throw new IOException("Cannot convert to long:" + datum.getClass());

    out.writeLong(num);
}
 
Example 30
Source Project: iceberg   Source File: ValueWriters.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void write(Map<K, V> map, Encoder encoder) throws IOException {
  encoder.writeMapStart();
  int numElements = map.size();
  encoder.setItemCount(numElements);
  Iterator<Map.Entry<K, V>> iter = map.entrySet().iterator();
  for (int i = 0; i < numElements; i += 1) {
    encoder.startItem();
    Map.Entry<K, V> entry = iter.next();
    keyWriter.write(entry.getKey(), encoder);
    valueWriter.write(entry.getValue(), encoder);
  }
  encoder.writeMapEnd();
}