Java Code Examples for org.apache.avro.io.DatumWriter#write()

The following examples show how to use org.apache.avro.io.DatumWriter#write() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JsonUtils.java From localization_nifi with Apache License 2.0

8 votes

/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream} as JSON.
 */
public static void write(GenericRecord record, OutputStream out) {
    try {
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
        JsonEncoder encoder = EncoderFactory.get().jsonEncoder(record.getSchema(), out);
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to read GenericRecord", e);
    }
}

Example 2

Source File: DataSerializer.java From jMetalSP with MIT License

6 votes

public byte[] serializeMessage(S clazz, String path) {

        byte[] result = null;
        try {

            File file = new File(path);

            Schema schema = new Schema.Parser().parse(file);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema);
            dataFileWriter.write(clazz, encoder);
            encoder.flush();
            result=out.toByteArray();
            out.close();
        }catch (Exception ex){
            ex.printStackTrace();
        }
        return result;
    }

Example 3

Source File: AvroSerializationSchema.java From pulsar with Apache License 2.0

6 votes

@Override
public byte[] serialize(T t) {
    if (null == t) {
        return null;
    }

    // Writer to serialize Avro record into a byte array.
    DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());
    // Output stream to serialize records into byte array.
    ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
    // Low-level class for serialization of Avro values.
    Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
    arrayOutputStream.reset();
    try {
        writer.write(t, encoder);
        encoder.flush();
    } catch (IOException e) {
        throw new RuntimeException("Error while serializing the record to Avro", e);
    }

    return arrayOutputStream.toByteArray();
}

Example 4

Source File: TransformTest.java From schema-registry-transfer-smt with Apache License 2.0

6 votes

private ByteArrayOutputStream encodeAvroObject(org.apache.avro.Schema schema, int sourceId, Object datum) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();

    out.write(MAGIC_BYTE);
    out.write(ByteBuffer.allocate(ID_SIZE).putInt(sourceId).array());

    EncoderFactory encoderFactory = EncoderFactory.get();
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
    Object
            value =
            datum instanceof NonRecordContainer ? ((NonRecordContainer) datum).getValue()
                    : datum;
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    writer.write(value, encoder);
    encoder.flush();

    return out;
}

Example 5

Source File: TruckParkAppIntegrationTest.java From data-highway with Apache License 2.0

5 votes

@Override
public byte[] serialize(String topic, Record record) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(DataDeserializer.MAGIC_BYTE);
    baos.write(Ints.toByteArray(VERSION));
    DatumWriter<Object> writer = new GenericDatumWriter<>(SCHEMA);
    BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(record, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException unreachable) {
    throw new RuntimeException(unreachable);
  }
}

Example 6

Source File: AvroCodec.java From schema-evolution-samples with Apache License 2.0

5 votes

@Override
public void encode(Object object, OutputStream outputStream) throws IOException {
	Schema schema = getSchema(object);
	Integer id = schemaRegistryClient.register(schema);
	DatumWriter writer = getDatumWriter(object.getClass(),schema);
	Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
	outputStream.write(ByteBuffer.allocate(4).putInt(id).array());
	writer.write(object,encoder);
	encoder.flush();
}

Example 7

Source File: AvroSerializer.java From spring-kafka with MIT License

5 votes

@Override
public byte[] serialize(String topic, T data) {
  try {
    byte[] result = null;

    if (data != null) {
      LOGGER.debug("data='{}'", data);

      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
      BinaryEncoder binaryEncoder =
          EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);

      DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(data.getSchema());
      datumWriter.write(data, binaryEncoder);

      binaryEncoder.flush();
      byteArrayOutputStream.close();

      result = byteArrayOutputStream.toByteArray();
      LOGGER.debug("serialized data='{}'", DatatypeConverter.printHexBinary(result));
    }
    return result;
  } catch (IOException ex) {
    throw new SerializationException(
        "Can't serialize data='" + data + "' for topic='" + topic + "'", ex);
  }
}

Example 8

Source File: AvroUtils.java From java-11-examples with Apache License 2.0

5 votes

public static byte[] serializeEmployee(Employee employee) throws IOException {
    DatumWriter<Employee> employeeDatumWriter = new SpecificDatumWriter<>(Employee.class);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(os, null);
    employeeDatumWriter.setSchema(Employee.getClassSchema());
    employeeDatumWriter.write(employee, binaryEncoder);
    os.flush();
    return os.toByteArray();
}

Example 9

Source File: FastDeserializerGeneratorForReuseTest.java From avro-util with BSD 2-Clause "Simplified" License

5 votes

private static byte[] serialize(GenericRecord record, Schema schema) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  Encoder encoder = AvroCompatibilityHelper.newBinaryEncoder(baos, true, null);
  DatumWriter datumWriter = new GenericDatumWriter(schema);
  datumWriter.write(record, encoder);
  encoder.flush();

  return baos.toByteArray();
}

Example 10

Source File: GenericRecordBinaryEncoder.java From gcp-ingestion with Mozilla Public License 2.0

5 votes

/**
 * Encode a GenericRecord into an avro encoded payload.
 */
public byte[] encodeRecord(GenericRecord record, Schema schema) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
    writer.write(record, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new UncheckedIOException(e);
  }
}

Example 11

Source File: AvroUtils.java From localization_nifi with Apache License 2.0

5 votes

/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream}.
 */
public static void write(GenericRecord record, OutputStream out) {
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
    try {
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to write AVRO record", e);
    }
}

Example 12

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_wrapSingleMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    runner.setProperty(ConvertAvroToJSON.WRAP_SINGLE_RECORD, Boolean.toString(true));
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Example 13

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_wrapSingleMessage() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_ARRAY);
    runner.setProperty(ConvertAvroToJSON.WRAP_SINGLE_RECORD, Boolean.toString(true));
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("[{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}]");
}

Example 14

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Example 15

Source File: TestConvertAvroToJSON.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testSingleSchemalessAvroMessage() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}

Example 16

Source File: OfframpIntegrationTest.java From data-highway with Apache License 2.0

5 votes

private static byte[] encode(Record record) throws IOException {
  try (ByteArrayOutputStream buffer = new ByteArrayOutputStream()) {
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(buffer, null);
    DatumWriter<Object> writer = new GenericDatumWriter<>(record.getSchema());
    buffer.write(0x00);
    buffer.write(Ints.toByteArray(1));
    writer.write(record, encoder);
    encoder.flush();
    return buffer.toByteArray();
  }
}

Example 17

Source File: DataDeserializerTest.java From data-highway with Apache License 2.0

5 votes

private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

Example 18

Source File: DataDeserializerGdprTest.java From data-highway with Apache License 2.0

5 votes

private byte[] toAvroBinary(Schema schema, Object value, int version) {
  try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
    baos.write(0x0);
    baos.write(Ints.toByteArray(version));
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().directBinaryEncoder(baos, null);
    writer.write(value, encoder);
    encoder.flush();
    return baos.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

Example 19

Source File: AvroKafkaSerializer.java From apicurio-registry with Apache License 2.0

5 votes

@Override
protected void serializeData(Schema schema, U data, OutputStream out) throws IOException {
    BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);

    // I guess this can happen if generics are lost with reflection ...
    if (data instanceof NonRecordContainer) {
        //noinspection unchecked
        data = (U) NonRecordContainer.class.cast(data).getValue();
    }

    DatumWriter<U> writer = avroDatumProvider.createDatumWriter(data, schema);
    writer.write(data, encoder);
    encoder.flush();
}

Example 20

Source File: TestAvroTranslator.java From envelope with Apache License 2.0

4 votes

private void performAvroTranslation(Config config) throws Exception {
  Schema schema = getSchema();

  Record record = new Record(schema);
  record.put("field1", "hello");
  record.put("field2", true);
  record.put("field3", ByteBuffer.wrap("world".getBytes()));
  record.put("field4", 1.0d);
  record.put("field5", 1);
  record.put("field6", 1.0f);
  record.put("field7", 1L);
  for (int i = 8; i <= 14; i++) {
    record.put("field" + i, null);
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
  DatumWriter<Record> writer = new GenericDatumWriter<>(schema);
  writer.write(record, encoder);
  encoder.flush();
  byte[] a = out.toByteArray();
  out.close();

  AvroTranslator t = new AvroTranslator();
  assertNoValidationFailures(t, config);
  t.configure(config);

  Row translated = t.translate(TestingMessageFactory.get(a, DataTypes.BinaryType)).iterator().next();

  Row validated = Contexts.getSparkSession().createDataFrame(
      Lists.newArrayList(translated), translated.schema()).collectAsList().get(0);

  assertEquals("hello", validated.getAs("field1"));
  assertEquals(true, validated.getAs("field2"));
  assertEquals("world", new String(validated.<byte[]>getAs("field3")));
  assertEquals(1.0d, validated.getAs("field4"));
  assertEquals(1, validated.getAs("field5"));
  assertEquals(1.0f, validated.getAs("field6"));
  assertEquals(1L, validated.getAs("field7"));
  for (int i = 8; i < 14; i++) {
    assertNull(validated.getAs("field" + i));
  }
}