Java Code Examples for org.apache.avro.io.Encoder#flush()
The following examples show how to use
org.apache.avro.io.Encoder#flush() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ParquetReader.java From reef with Apache License 2.0 | 7 votes |
/** * Serialize Avro data to a in-memory ByteBuffer. * @return A ByteBuffer that contains avro data. * @throws IOException if the parquet file couldn't be parsed correctly. */ public ByteBuffer serializeToByteBuffer() throws IOException { final ByteArrayOutputStream stream = new ByteArrayOutputStream(); final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null); final DatumWriter writer = new GenericDatumWriter<GenericRecord>(); writer.setSchema(createAvroSchema()); final AvroParquetReader<GenericRecord> reader = createAvroReader(); GenericRecord record = reader.read(); while (record != null) { writer.write(record, encoder); record = reader.read(); } try { reader.close(); } catch (IOException ex){ LOG.log(Level.SEVERE, ex.getMessage()); throw ex; } encoder.flush(); final ByteBuffer buf = ByteBuffer.wrap(stream.toByteArray()); buf.order(ByteOrder.LITTLE_ENDIAN); return buf; }
Example 2
Source File: BigQueryIOStorageQueryTest.java From beam with Apache License 2.0 | 6 votes |
private static ReadRowsResponse createResponse( Schema schema, Collection<GenericRecord> genericRecords, double fractionConsumed) throws Exception { GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder binaryEncoder = ENCODER_FACTORY.binaryEncoder(outputStream, null); for (GenericRecord genericRecord : genericRecords) { writer.write(genericRecord, binaryEncoder); } binaryEncoder.flush(); return ReadRowsResponse.newBuilder() .setAvroRows( AvroRows.newBuilder() .setSerializedBinaryRows(ByteString.copyFrom(outputStream.toByteArray())) .setRowCount(genericRecords.size())) .setStatus(StreamStatus.newBuilder().setFractionConsumed((float) fractionConsumed)) .build(); }
Example 3
Source File: RegistryAvroSerializationSchema.java From flink with Apache License 2.0 | 6 votes |
@Override public byte[] serialize(T object) { checkAvroInitialized(); if (object == null) { return null; } else { try { ByteArrayOutputStream outputStream = getOutputStream(); outputStream.reset(); Encoder encoder = getEncoder(); schemaCoder.writeSchema(getSchema(), outputStream); getDatumWriter().write(object, encoder); encoder.flush(); return outputStream.toByteArray(); } catch (IOException e) { throw new WrappingRuntimeException("Failed to serialize schema registry.", e); } } }
Example 4
Source File: BigQueryIOStorageReadTest.java From beam with Apache License 2.0 | 6 votes |
private static ReadRowsResponse createResponse( Schema schema, Collection<GenericRecord> genericRecords, double fractionConsumed) throws Exception { GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder binaryEncoder = ENCODER_FACTORY.binaryEncoder(outputStream, null); for (GenericRecord genericRecord : genericRecords) { writer.write(genericRecord, binaryEncoder); } binaryEncoder.flush(); return ReadRowsResponse.newBuilder() .setAvroRows( AvroRows.newBuilder() .setSerializedBinaryRows(ByteString.copyFrom(outputStream.toByteArray())) .setRowCount(genericRecords.size())) .setStatus(StreamStatus.newBuilder().setFractionConsumed((float) fractionConsumed)) .build(); }
Example 5
Source File: FastGenericSerializerGeneratorTest.java From avro-util with BSD 2-Clause "Simplified" License | 6 votes |
public <T> Decoder dataAsBinaryDecoder(T data, Schema schema) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); //new BinaryEncoder(baos); try { FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null); FastSerializer<T> fastSerializer = fastGenericSerializerGenerator.generateSerializer(); fastSerializer.serialize(data, binaryEncoder); binaryEncoder.flush(); } catch (Exception e) { throw new RuntimeException(e); } return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null); }
Example 6
Source File: FastSpecificSerializerGeneratorTest.java From avro-util with BSD 2-Clause "Simplified" License | 6 votes |
public <T> Decoder dataAsDecoder(T data, Schema schema) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); try { FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null); FastSerializer<T> fastSerializer = fastSpecificSerializerGenerator.generateSerializer(); fastSerializer.serialize(data, binaryEncoder); binaryEncoder.flush(); } catch (Exception e) { throw new RuntimeException(e); } return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null); }
Example 7
Source File: FastStringableTest.java From avro-util with BSD 2-Clause "Simplified" License | 6 votes |
public <T> Decoder writeWithFastAvro(T data, Schema schema, boolean specific) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); try { FastSerializer<T> fastSerializer; if (specific) { FastSpecificSerializerGenerator<T> fastSpecificSerializerGenerator = new FastSpecificSerializerGenerator<>(schema, tempDir, classLoader, null); fastSerializer = fastSpecificSerializerGenerator.generateSerializer(); } else { FastGenericSerializerGenerator<T> fastGenericSerializerGenerator = new FastGenericSerializerGenerator<>(schema, tempDir, classLoader, null); fastSerializer = fastGenericSerializerGenerator.generateSerializer(); } fastSerializer.serialize(data, binaryEncoder); binaryEncoder.flush(); } catch (Exception e) { throw new RuntimeException(e); } return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null); }
Example 8
Source File: AvroUtils.java From brooklin with BSD 2-Clause "Simplified" License | 5 votes |
private static byte[] encodeAvroIndexedRecord(Schema schema, IndexedRecord record, ByteArrayOutputStream outputStream, Encoder encoder) throws IOException { DatumWriter<IndexedRecord> datumWriter; if (record instanceof GenericRecord) { datumWriter = new GenericDatumWriter<>(schema); } else { datumWriter = new SpecificDatumWriter<>(schema); } datumWriter.write(record, encoder); encoder.flush(); outputStream.close(); return outputStream.toByteArray(); }
Example 9
Source File: DataDeserializerGdprTest.java From data-highway with Apache License 2.0 | 5 votes |
private byte[] toAvroBinary(Schema schema, Object value, int version) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { baos.write(0x0); baos.write(Ints.toByteArray(version)); DatumWriter<Object> writer = new GenericDatumWriter<>(schema); Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null); writer.write(value, encoder); encoder.flush(); return baos.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
Example 10
Source File: Utils.java From kafka-monitor with Apache License 2.0 | 5 votes |
public static String jsonFromGenericRecord(GenericRecord record) { ByteArrayOutputStream out = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(DefaultTopicSchema.MESSAGE_V0); try { Encoder encoder = new JsonEncoder(DefaultTopicSchema.MESSAGE_V0, out); writer.write(record, encoder); encoder.flush(); } catch (IOException e) { LOG.error("Unable to serialize avro record due to error " + e); } return out.toString(); }
Example 11
Source File: AvroSupport.java From javabase with Apache License 2.0 | 5 votes |
public static byte[] dataToByteArray(Schema schema, GenericRecord datum) throws IOException { GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema); ByteArrayOutputStream os = new ByteArrayOutputStream(); try { Encoder e = EncoderFactory.get().binaryEncoder(os, null); writer.write(datum, e); e.flush(); byte[] byteData = os.toByteArray(); return byteData; } finally { os.close(); } }
Example 12
Source File: RecordBenchmarkBase.java From avro-fastserde with Apache License 2.0 | 5 votes |
@Setup public void init() throws Exception { final GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(specificRecordSchema); for (int i = 0; i < 1000; i++) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); genericRecords.add(FastSerdeBenchmarkSupport.generateRandomRecordData(specificRecordSchema)); specificRecords .add(FastSerdeBenchmarkSupport.toSpecificRecord(genericRecords.get(genericRecords.size() - 1))); datumWriter.write(genericRecords.get(genericRecords.size() - 1), encoder); encoder.flush(); recordBytes.add(baos.toByteArray()); } fastGenericDatumReader = new FastGenericDatumReader<>( specificRecordSchema, cache); fastGenericDatumWriter = new FastGenericDatumWriter<>(specificRecordSchema, cache); genericDatumReader = new GenericDatumReader<>(specificRecordSchema); genericDatumWriter = new GenericDatumWriter<>(specificRecordSchema); fastSpecificDatumReader = new FastSpecificDatumReader<>( specificRecordSchema, cache); fastSpecificDatumWriter = new FastSpecificDatumWriter<>(specificRecordSchema, cache); specificDatumReader = new SpecificDatumReader<>(specificRecordSchema); specificDatumWriter = new SpecificDatumWriter<>(specificRecordSchema); }
Example 13
Source File: FastSerdeBenchmarkSupport.java From avro-fastserde with Apache License 2.0 | 5 votes |
public static <T extends SpecificRecord> T toSpecificRecord(GenericData.Record record) throws IOException { GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(record.getSchema()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null); datumWriter.write(record, binaryEncoder); binaryEncoder.flush(); SpecificDatumReader<T> datumReader = new SpecificDatumReader<>(record.getSchema()); return datumReader.read(null, DecoderFactory.get().binaryDecoder(baos.toByteArray(), null)); }
Example 14
Source File: AvroSerializer.java From secor with Apache License 2.0 | 5 votes |
public static byte[] serialize(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length); serialized.put(out.toByteArray()); return serialized.array(); }
Example 15
Source File: FastSerdeTestsSupport.java From avro-util with BSD 2-Clause "Simplified" License | 5 votes |
public static <T> Decoder specificDataAsDecoder(T record, Schema schema) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null); try { SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema); writer.write(record, binaryEncoder); binaryEncoder.flush(); } catch (Exception e) { throw new RuntimeException(e); } return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null); }
Example 16
Source File: GenericRecordBinaryEncoder.java From gcp-ingestion with Mozilla Public License 2.0 | 5 votes |
/** * Encode a GenericRecord into an avro encoded payload. */ public byte[] encodeRecord(GenericRecord record, Schema schema) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); writer.write(record, encoder); encoder.flush(); return baos.toByteArray(); } catch (IOException e) { throw new UncheckedIOException(e); } }
Example 17
Source File: WatcherAvroUtil.java From reef with Apache License 2.0 | 5 votes |
public static String toString(final SpecificRecord record) { final String jsonEncodedRecord; try { final Schema schema = record.getSchema(); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); final Encoder encoder = EncoderFactory.get().jsonEncoder(schema, bos); final SpecificDatumWriter datumWriter = new SpecificDatumWriter(record.getClass()); datumWriter.write(record, encoder); encoder.flush(); jsonEncodedRecord = new String(bos.toByteArray(), Charset.forName("UTF-8")); } catch (final IOException e) { throw new RuntimeException(e); } return jsonEncodedRecord; }
Example 18
Source File: DataDeserializerTest.java From data-highway with Apache License 2.0 | 5 votes |
private byte[] toAvroBinary(Schema schema, Object value, int version) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { baos.write(0x0); baos.write(Ints.toByteArray(version)); DatumWriter<Object> writer = new GenericDatumWriter<>(schema); Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null); writer.write(value, encoder); encoder.flush(); return baos.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
Example 19
Source File: AvroUtils.java From kite with Apache License 2.0 | 3 votes |
/** * Given an entity, an avro schema, and an encoder, write the entity to the * encoder's underlying output stream. * * @param entity * The entity we want to encode. * @param encoder * The Avro Encoder we will write to. * @param writer * The DatumWriter we'll use to encode the entity to the encoder. */ public static <T> void writeAvroEntity(T entity, Encoder encoder, DatumWriter<T> writer) { try { writer.write(entity, encoder); encoder.flush(); } catch (IOException e) { throw new SerializationException("Could not serialize Avro entity", e); } }
Example 20
Source File: GenerateKafkaEvents.java From samza with Apache License 2.0 | 3 votes |
/** * Encode an Avro record into byte array * * @param clazz The class type of the avro record * @param record the instance of the avro record * @param <T> The type of the avro record. * @return encoded bytes * @throws IOException on I/O errors encoding the avro record */ public static <T> byte[] encodeAvroSpecificRecord(Class<T> clazz, T record) throws IOException { DatumWriter<T> msgDatumWriter = new SpecificDatumWriter<>(clazz); ByteArrayOutputStream os = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(os, null); msgDatumWriter.write(record, encoder); encoder.flush(); return os.toByteArray(); }