Java Code Examples for org.apache.avro.specific.SpecificDatumWriter#write()

The following examples show how to use org.apache.avro.specific.SpecificDatumWriter#write() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: GenericAvroSerializer.java From geowave with Apache License 2.0

6 votes

public static synchronized <T> byte[] serialize(final T avroObject, final Schema avroSchema) {

    try {
      final ByteArrayOutputStream os = new ByteArrayOutputStream();
      final BinaryEncoder encoder = ef.binaryEncoder(os, null);

      final String schemaName = getSchemaName(avroSchema);
      if (!writers.containsKey(schemaName)) {
        writers.put(schemaName, new SpecificDatumWriter<T>(avroSchema));
      }

      final SpecificDatumWriter<T> writer = writers.get(schemaName);
      writer.write(avroObject, encoder);
      encoder.flush();
      return os.toByteArray();
    } catch (final IOException e) {
      LOGGER.error("Unable to serialize Avro record to byte[]: " + e.getMessage(), e);
      return null;
    }
  }

Example 2

Source File: TypeUtils.java From geowave with Apache License 2.0

6 votes

private static <T> byte[] deserialize(
    final T avroObject,
    final Schema avroSchema,
    final Class<T> avroClass) throws IOException {

  final ByteArrayOutputStream os = new ByteArrayOutputStream();
  final BinaryEncoder encoder = ef.binaryEncoder(os, null);
  if (!writers.containsKey(avroClass.toString())) {
    writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema));
  }

  final SpecificDatumWriter<T> writer = writers.get(avroClass.toString());
  writer.write(avroObject, encoder);
  encoder.flush();
  return os.toByteArray();
}

Example 3

Source File: AvroLWM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License

6 votes

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}

Example 4

Source File: AvroOneM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License

6 votes

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}

Example 5

Source File: AvroLWM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License

6 votes

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}

Example 6

Source File: AvroOneM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License

6 votes

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}

Example 7

Source File: JavaToAvroSerializer.java From Decision with Apache License 2.0

6 votes

private byte[] getInsertMessageBytes(InsertMessage insertMessage){

        byte[] result = null;

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

        try {
            writer.write(insertMessage, encoder);
            encoder.flush();
            out.close();
            result = out.toByteArray();
        }catch (IOException e){
            return null;
        }

        return result;

    }

Example 8

Source File: HardwareEmulatorMain.java From Decision with Apache License 2.0

6 votes

private byte[] getInsertMessageBytes(InsertMessage insertMessage){

            byte[] result = null;

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

            try {
                writer.write(insertMessage, encoder);
                encoder.flush();
                out.close();
                result = out.toByteArray();
            }catch (IOException e){
                return null;
            }

            return result;

        }

Example 9

Source File: AvroRowDataDeSerializationSchemaTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSpecificType() throws Exception {
	JodaTimeRecord record = new JodaTimeRecord();
	record.setTypeTimestampMillis(DateTime.parse("2010-06-30T01:20:20"));
	record.setTypeDate(LocalDate.parse("2014-03-01"));
	record.setTypeTimeMillis(LocalTime.parse("12:12:12"));
	SpecificDatumWriter<JodaTimeRecord> datumWriter = new SpecificDatumWriter<>(JodaTimeRecord.class);
	ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
	Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);
	datumWriter.write(record, encoder);
	encoder.flush();
	byte[] input = byteArrayOutputStream.toByteArray();

	DataType dataType = ROW(
			FIELD("type_timestamp_millis", TIMESTAMP(3)),
			FIELD("type_date", DATE()),
			FIELD("type_time_millis", TIME(3)));
	final RowType rowType = (RowType) dataType.getLogicalType();
	final TypeInformation<RowData> typeInfo = new RowDataTypeInfo(rowType);
	AvroRowDataSerializationSchema serializationSchema = new AvroRowDataSerializationSchema(rowType);
	serializationSchema.open(null);
	AvroRowDataDeserializationSchema deserializationSchema =
			new AvroRowDataDeserializationSchema(rowType, typeInfo);
	deserializationSchema.open(null);

	RowData rowData = deserializationSchema.deserialize(input);
	byte[] output = serializationSchema.serialize(rowData);
	RowData rowData2 = deserializationSchema.deserialize(output);
	Assert.assertEquals(rowData, rowData2);
	Assert.assertEquals("2010-06-30T01:20:20", rowData.getTimestamp(0, 3).toString());
	Assert.assertEquals("2014-03-01", DataFormatConverters.LocalDateConverter.INSTANCE.toExternal(
			rowData.getInt(1)).toString());
	Assert.assertEquals("12:12:12", DataFormatConverters.LocalTimeConverter.INSTANCE.toExternal(
			rowData.getInt(2)).toString());
}

Example 10

Source File: RecordTest.java From gradle-avro-plugin with Apache License 2.0

5 votes

@Disabled
@ParameterizedTest
@MethodSource
<T extends SpecificRecord> void buildAndWriteRecord(T record) throws Exception {
    Schema schema = record.getSchema();
    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>();
    OutputStream outputStream = new ByteArrayOutputStream();
    Encoder jsonEncoder = encoderFactory.jsonEncoder(schema, outputStream, true);
    Encoder validatingEncoder = encoderFactory.validatingEncoder(schema, jsonEncoder);
    writer.write(record, validatingEncoder);
}

Example 11

Source File: WatcherAvroUtil.java From reef with Apache License 2.0

5 votes

public static String toString(final SpecificRecord record) {
  final String jsonEncodedRecord;
  try {
    final Schema schema = record.getSchema();
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Encoder encoder = EncoderFactory.get().jsonEncoder(schema, bos);
    final SpecificDatumWriter datumWriter = new SpecificDatumWriter(record.getClass());
    datumWriter.write(record, encoder);
    encoder.flush();
    jsonEncodedRecord = new String(bos.toByteArray(), Charset.forName("UTF-8"));
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
  return jsonEncodedRecord;
}

Example 12

Source File: AvroSerializer.java From secor with Apache License 2.0

5 votes

public static byte[] serialize(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
    writer.write(record, encoder);
    encoder.flush();
    ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length);
    serialized.put(out.toByteArray());
    return serialized.array();
}

Example 13

Source File: AvroCompatibilityHelperGeneratedEnumClassesTest.java From avro-util with BSD 2-Clause "Simplified" License

5 votes

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}

Example 14

Source File: FastSerdeTestsSupport.java From avro-fastserde with Apache License 2.0

5 votes

public static <T> Decoder serializeSpecific(T record, Schema schema) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(baos, null);

    try {
        SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
        writer.write(record, binaryEncoder);
        binaryEncoder.flush();

    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return DecoderFactory.get().binaryDecoder(baos.toByteArray(), null);
}

Example 15

Source File: FastSerdeTestsSupport.java From avro-util with BSD 2-Clause "Simplified" License

5 votes

public static <T> Decoder specificDataAsDecoder(T record, Schema schema) {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);

  try {
    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
    writer.write(record, binaryEncoder);
    binaryEncoder.flush();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  return DecoderFactory.defaultFactory().createBinaryDecoder(baos.toByteArray(), null);
}

Example 16

Source File: AvroCompatibilityHelperGeneratedFixedClassesTest.java From avro-util with BSD 2-Clause "Simplified" License

5 votes

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}

Example 17

Source File: HBaseWriter.java From hiped2 with Apache License 2.0

4 votes

/**
 * The MapReduce driver - setup and launch the job.
 *
 * @param args the command-line arguments
 * @return the process exit code
 * @throws Exception if something goes wrong
 */
public int run(final String[] args) throws Exception {

  Cli cli = Cli.builder().setArgs(args).addOptions(CliCommonOpts.InputFileOption.values()).build();
  int result = cli.runCmd();

  if (result != 0) {
    return result;
  }

  File inputFile = new File(cli.getArgValueAsString(CliCommonOpts.InputFileOption.INPUT));

  Configuration conf = HBaseConfiguration.create();

  createTableAndColumn(conf, STOCKS_TABLE_NAME,
      STOCK_DETAILS_COLUMN_FAMILY_AS_BYTES);

  HTable htable = new HTable(conf, STOCKS_TABLE_NAME);
  htable.setAutoFlush(false);
  htable.setWriteBufferSize(1024 * 1024 * 12);


  SpecificDatumWriter<Stock> writer =
      new SpecificDatumWriter<Stock>();
  writer.setSchema(Stock.SCHEMA$);

  ByteArrayOutputStream bao = new ByteArrayOutputStream();
  BinaryEncoder encoder =
      EncoderFactory.get().directBinaryEncoder(bao, null);

  for (Stock stock: AvroStockUtils.fromCsvFile(inputFile)) {
    writer.write(stock, encoder);
    encoder.flush();

    byte[] rowkey = Bytes.add(
        Bytes.toBytes(stock.getSymbol().toString()),
        Bytes.toBytes(stock.getDate().toString()));

    byte[] stockAsAvroBytes = bao.toByteArray();

    Put put = new Put(rowkey);
    put.add(STOCK_DETAILS_COLUMN_FAMILY_AS_BYTES,
        STOCK_COLUMN_QUALIFIER_AS_BYTES,
        stockAsAvroBytes);

    htable.put(put);

    bao.reset();
  }

  htable.flushCommits();
  htable.close();
  System.out.println("done");

  return 0;
}