Java Code Examples for org.apache.avro.io.Encoder#flush()

The following examples show how to use org.apache.avro.io.Encoder#flush() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ParquetReader.java    From reef with Apache License 2.0 7 votes vote down vote up
/**
 * Serialize Avro data to a in-memory ByteBuffer.
 * @return A ByteBuffer that contains avro data.
 * @throws IOException if the parquet file couldn't be parsed correctly.
 */
public ByteBuffer serializeToByteBuffer() throws IOException {
  final ByteArrayOutputStream stream = new ByteArrayOutputStream();
  final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null);
  final DatumWriter writer = new GenericDatumWriter<GenericRecord>();
  writer.setSchema(createAvroSchema());
  final AvroParquetReader<GenericRecord> reader = createAvroReader();

  GenericRecord record = reader.read();
  while (record != null) {
    writer.write(record, encoder);
    record = reader.read();
  }

  try {
    reader.close();
  } catch (IOException ex){
    LOG.log(Level.SEVERE, ex.getMessage());
    throw ex;
  }

  encoder.flush();
  final ByteBuffer buf = ByteBuffer.wrap(stream.toByteArray());
  buf.order(ByteOrder.LITTLE_ENDIAN);
  return buf;
}
 
Example 2
Source File: BigQueryIOStorageQueryTest.java    From beam with Apache License 2.0 6 votes vote down vote up
private static ReadRowsResponse createResponse(
    Schema schema, Collection<GenericRecord> genericRecords, double fractionConsumed)
    throws Exception {
  GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  Encoder binaryEncoder = ENCODER_FACTORY.binaryEncoder(outputStream, null);
  for (GenericRecord genericRecord : genericRecords) {
    writer.write(genericRecord, binaryEncoder);
  }

  binaryEncoder.flush();

  return ReadRowsResponse.newBuilder()
      .setAvroRows(
          AvroRows.newBuilder()
              .setSerializedBinaryRows(ByteString.copyFrom(outputStream.toByteArray()))
              .setRowCount(genericRecords.size()))
      .setStatus(StreamStatus.newBuilder().setFractionConsumed((float) fractionConsumed))
      .build();
}
 
Example 3
Source File: RegistryAvroSerializationSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] serialize(T object) {
	checkAvroInitialized();

	if (object == null) {
		return null;
	} else {
		try {
			ByteArrayOutputStream outputStream = getOutputStream();
			outputStream.reset();
			Encoder encoder = getEncoder();
			schemaCoder.writeSchema(getSchema(), outputStream);
			getDatumWriter().write(object, encoder);
			encoder.flush();
			return outputStream.toByteArray();
		} catch (IOException e) {
			throw new WrappingRuntimeException("Failed to serialize schema registry.", e);
		}
	}
}
 
Example 4
Source File: BigQueryIOStorageReadTest.java    From beam with Apache License 2.0 6 votes vote down vote up
private static ReadRowsResponse createResponse(
    Schema schema, Collection<GenericRecord> genericRecords, double fractionConsumed)
    throws Exception {
  GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  Encoder binaryEncoder = ENCODER_FACTORY.binaryEncoder(outputStream, null);
  for (GenericRecord genericRecord : genericRecords) {
    writer.write(genericRecord, binaryEncoder);
  }

  binaryEncoder.flush();

  return ReadRowsResponse.newBuilder()
      .setAvroRows(
          AvroRows.newBuilder()
              .setSerializedBinaryRows(ByteString.copyFrom(outputStream.toByteArray()))
              .setRowCount(genericRecords.size()))
      .setStatus(StreamStatus.newBuilder().setFractionConsumed((float) fractionConsumed))
      .build();
}
 
Example 5
Source File: FastGenericSerializerGeneratorTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public <T> Decoder dataAsBinaryDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); //new BinaryEncoder(baos);

  try {
    FastGenericSerializerGenerator<T> fastGenericSerializerGenerator =
        new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 6
Source File: FastSpecificSerializerGeneratorTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public <T> Decoder dataAsDecoder(T data, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator =
        new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
    FastSerializer<T> fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 7
Source File: FastStringableTest.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public <T> Decoder writeWithFastAvro(T data, Schema schema, boolean specific) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    FastSerializer<T> fastSerializer;
    if (specific) {
      FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastSpecificSerializerGenerator.generateSerializer();
    } else {
      FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null);
      fastSerializer = fastGenericSerializerGenerator.generateSerializer();
    }
    fastSerializer.serialize(data, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 8
Source File: AvroUtils.java    From brooklin with BSD 2-Clause "Simplified" License 5 votes vote down vote up
private static byte[] encodeAvroIndexedRecord(Schema schema, IndexedRecord record,
    ByteArrayOutputStream outputStream, Encoder encoder) throws IOException {
  DatumWriter<IndexedRecord> datumWriter;
  if (record instanceof GenericRecord) {
    datumWriter = new GenericDatumWriter<>(schema);
  } else {
    datumWriter = new SpecificDatumWriter<>(schema);
  }
  datumWriter.write(record, encoder);
  encoder.flush();
  outputStream.close();
  return outputStream.toByteArray();
}
 
Example 9
Source File: DataDeserializerGdprTest.java    From data-highway with Apache License 2.0 5 votes vote down vote up
private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 10
Source File: Utils.java    From kafka-monitor with Apache License 2.0 5 votes vote down vote up
public static String jsonFromGenericRecord(GenericRecord record) {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(DefaultTopicSchema.MESSAGE_V0);

  try {
    Encoder encoder = new JsonEncoder(DefaultTopicSchema.MESSAGE_V0, out);
    writer.write(record, encoder);
    encoder.flush();
  } catch (IOException e) {
    LOG.error("Unable to serialize avro record due to error " + e);
  }
  return out.toString();
}
 
Example 11
Source File: AvroSupport.java    From javabase with Apache License 2.0 5 votes vote down vote up
public static byte[] dataToByteArray(Schema schema, GenericRecord datum) throws IOException {
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    try {
        Encoder e = EncoderFactory.get().binaryEncoder(os, null);
        writer.write(datum, e);
        e.flush();
        byte[] byteData = os.toByteArray();
        return byteData;
    } finally {
        os.close();
    }
}
 
Example 12
Source File: RecordBenchmarkBase.java    From avro-fastserde with Apache License 2.0 5 votes vote down vote up
@Setup
public void init() throws Exception {
    final GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(specificRecordSchema);
    for (int i = 0; i < 1000; i++) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);

        genericRecords.add(FastSerdeBenchmarkSupport.generateRandomRecordData(specificRecordSchema));
        specificRecords
                .add(FastSerdeBenchmarkSupport.toSpecificRecord(genericRecords.get(genericRecords.size() - 1)));

        datumWriter.write(genericRecords.get(genericRecords.size() - 1), encoder);
        encoder.flush();

        recordBytes.add(baos.toByteArray());
    }
    fastGenericDatumReader = new FastGenericDatumReader<>(
            specificRecordSchema, cache);
    fastGenericDatumWriter = new FastGenericDatumWriter<>(specificRecordSchema, cache);

    genericDatumReader = new GenericDatumReader<>(specificRecordSchema);
    genericDatumWriter = new GenericDatumWriter<>(specificRecordSchema);

    fastSpecificDatumReader = new FastSpecificDatumReader<>(
            specificRecordSchema, cache);
    fastSpecificDatumWriter = new FastSpecificDatumWriter<>(specificRecordSchema, cache);

    specificDatumReader = new SpecificDatumReader<>(specificRecordSchema);
    specificDatumWriter = new SpecificDatumWriter<>(specificRecordSchema);
}
 
Example 13
Source File: FastSerdeBenchmarkSupport.java    From avro-fastserde with Apache License 2.0 5 votes vote down vote up
public static <T extends SpecificRecord> T toSpecificRecord(GenericData.Record record) throws IOException {
    GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(record.getSchema());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();

    Encoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null);
    datumWriter.write(record, binaryEncoder);
    binaryEncoder.flush();

    SpecificDatumReader<T> datumReader = new SpecificDatumReader<>(record.getSchema());
    return datumReader.read(null, DecoderFactory.get().binaryDecoder(baos.toByteArray(), null));
}
 
Example 14
Source File: AvroSerializer.java    From secor with Apache License 2.0 5 votes vote down vote up
public static byte[] serialize(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
    writer.write(record, encoder);
    encoder.flush();
    ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length);
    serialized.put(out.toByteArray());
    return serialized.array();
}
 
Example 15
Source File: FastSerdeTestsSupport.java    From avro-util with BSD 2-Clause "Simplified" License 5 votes vote down vote up
public static <T> Decoder specificDataAsDecoder(T record, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
    writer.write(record, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}
 
Example 16
Source File: GenericRecordBinaryEncoder.java    From gcp-ingestion with Mozilla Public License 2.0 5 votes vote down vote up
/**
 * Encode a GenericRecord into an avro encoded payload.
 */
public byte[] encodeRecord(GenericRecord record, Schema schema) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
    writer.write(record, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}
 
Example 17
Source File: WatcherAvroUtil.java    From reef with Apache License 2.0 5 votes vote down vote up
public static String toString(final SpecificRecord record) {
  final String jsonEncodedRecord;
  try {
    final Schema schema = record.getSchema();
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Encoder encoder = EncoderFactory.get().jsonEncoder(schema, bos);
    final SpecificDatumWriter datumWriter = new SpecificDatumWriter(record.getClass());
    datumWriter.write(record, encoder);
    encoder.flush();
    jsonEncodedRecord = new String(bos.toByteArray(), Charset.forName("UTF-8"));
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
  return jsonEncodedRecord;
}
 
Example 18
Source File: DataDeserializerTest.java    From data-highway with Apache License 2.0 5 votes vote down vote up
private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 19
Source File: AvroUtils.java    From kite with Apache License 2.0 3 votes vote down vote up
/**
 * Given an entity, an avro schema, and an encoder, write the entity to the
 * encoder's underlying output stream.
 * 
 * @param entity
 *          The entity we want to encode.
 * @param encoder
 *          The Avro Encoder we will write to.
 * @param writer
 *          The DatumWriter we'll use to encode the entity to the encoder.
 */
public static <T> void writeAvroEntity(T entity, Encoder encoder,
    DatumWriter<T> writer) {
  try {
    writer.write(entity, encoder);
    encoder.flush();
  } catch (IOException e) {
    throw new SerializationException("Could not serialize Avro entity", e);
  }
}
 
Example 20
Source File: GenerateKafkaEvents.java    From samza with Apache License 2.0 3 votes vote down vote up
/**
 * Encode an Avro record into byte array
 *
 * @param clazz The class type of the avro record
 * @param record the instance of the avro record
 * @param <T> The type of the avro record.
 * @return encoded bytes
 * @throws IOException on I/O errors encoding the avro record
 */
public static <T> byte[] encodeAvroSpecificRecord(Class<T> clazz, T record) throws IOException {
  DatumWriter<T> msgDatumWriter = new SpecificDatumWriter<>(clazz);
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  Encoder encoder = EncoderFactory.get().binaryEncoder(os, null);
  msgDatumWriter.write(record, encoder);
  encoder.flush();
  return os.toByteArray();
}