org.apache.avro.specific.SpecificDatumWriter Java Examples

@VisibleForTesting
byte[] encodeRecord(IndexedRecord record) {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  try {
    EncoderFactory encoderfactory = new EncoderFactory();
    BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
    DatumWriter<IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(schema);
    } else {
      writer = new GenericDatumWriter<>(schema);
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (Exception e) {
    throw new SamzaException("Unable to serialize Avro record using schema within the record: " + schema.toString(), e);
  }
  return out.toByteArray();
}
 

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 

@Override
public byte[] encode(final EvaluatorControl evaluatorControl) {
  try {
    LOG.log(Level.FINEST, "Before Encoding: {0}", evaluatorControl.toString());
    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    final DatumWriter<EvaluatorControl> writer = new SpecificDatumWriter<>(EvaluatorControl.getClassSchema());
    writer.write(evaluatorControl, encoder);
    encoder.flush();
    out.close();
    LOG.log(Level.FINEST, "After Encoding");
    return out.toByteArray();
  } catch (final IOException ex) {
    throw new RemoteRuntimeException(ex);
  }
}
 

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 

public byte[] serializeMessage(S clazz, String path) {

        byte[] result = null;
        try {

            File file = new File(path);

            Schema schema = new Schema.Parser().parse(file);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema);
            dataFileWriter.write(clazz, encoder);
            encoder.flush();
            result=out.toByteArray();
            out.close();
        }catch (Exception ex){
            ex.printStackTrace();
        }
        return result;
    }
 

private void writeAvroJobSpec(AvroJobSpec jobSpec) throws IOException {
  DatumWriter<AvroJobSpec> datumWriter = new SpecificDatumWriter<>(AvroJobSpec.SCHEMA$);
  DataFileWriter<AvroJobSpec> dataFileWriter = new DataFileWriter<>(datumWriter);

  Path jobSpecPath = new Path(this.specConsumerPath, jobSpec.getUri());

  //Write the new JobSpec to a temporary path first.
  Path tmpDir = new Path(this.specConsumerPath, "_tmp");
  if (!fs.exists(tmpDir)) {
    fs.mkdirs(tmpDir);
  }

  Path tmpJobSpecPath = new Path(tmpDir, jobSpec.getUri());

  OutputStream out = fs.create(tmpJobSpecPath);

  dataFileWriter.create(AvroJobSpec.SCHEMA$, out);
  dataFileWriter.append(jobSpec);
  dataFileWriter.close();

  //Rename the JobSpec from temporary to final location.
  HadoopUtils.renamePath(fs, tmpJobSpecPath, jobSpecPath, true);
}
 

protected void checkAvroInitialized() {
	if (datumWriter != null) {
		return;
	}
	ClassLoader cl = Thread.currentThread().getContextClassLoader();
		if (SpecificRecord.class.isAssignableFrom(recordClazz)) {
		Schema schema = SpecificData.get().getSchema(recordClazz);
		this.datumWriter = new SpecificDatumWriter<>(schema);
		this.schema = schema;
	} else {
		this.schema = new Schema.Parser().parse(this.schemaString);
		GenericData genericData = new GenericData(cl);

		this.datumWriter = new GenericDatumWriter<>(schema, genericData);
	}
	this.arrayOutputStream = new ByteArrayOutputStream();
	this.encoder = EncoderFactory.get().directBinaryEncoder(arrayOutputStream, null);
}
 

@Override
public byte[] serialize(T t) {
    if (null == t) {
        return null;
    }

    // Writer to serialize Avro record into a byte array.
    DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());
    // Output stream to serialize records into byte array.
    ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
    // Low-level class for serialization of Avro values.
    Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
    arrayOutputStream.reset();
    try {
        writer.write(t, encoder);
        encoder.flush();
    } catch (IOException e) {
        throw new RuntimeException("Error while serializing the record to Avro", e);
    }

    return arrayOutputStream.toByteArray();
}
 

private static <T> byte[] deserialize(
    final T avroObject,
    final Schema avroSchema,
    final Class<T> avroClass) throws IOException {

  final ByteArrayOutputStream os = new ByteArrayOutputStream();
  final BinaryEncoder encoder = ef.binaryEncoder(os, null);
  if (!writers.containsKey(avroClass.toString())) {
    writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema));
  }

  final SpecificDatumWriter<T> writer = writers.get(avroClass.toString());
  writer.write(avroObject, encoder);
  encoder.flush();
  return os.toByteArray();
}
 

/**
 * Produce a JSON string that represents given configuration.
 * @param configuration Tang configuration to convert into a JSON string.
 * @param prettyPrint If true, use new lines and spaces to pretty print the JSON string.
 * If false (by default), output JSON as a single line.
 * @return A JSON string that corresponds to the given Tang configuration.
 */
public String toString(final Configuration configuration, final boolean prettyPrint) {
  final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class);
  final String result;
  try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
    // TODO [REEF-1536] Re-enable pretty printing when Avro 1.7.5 available on all environments:
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(AvroConfiguration.SCHEMA$, out); //, prettyPrint);
    configurationWriter.write(toAvro(configuration), encoder);
    encoder.flush();
    out.flush();
    result = out.toString(JSON_CHARSET);
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
  return result;
}
 

static void writeAvroYarnJobSubmissionParametersToOutputStream(
    final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS,
    final String jobFolderOnDFSPath,
    final OutputStream outputStream) throws IOException {
  final DatumWriter<AvroYarnJobSubmissionParameters> datumWriter =
      new SpecificDatumWriter<>(AvroYarnJobSubmissionParameters.class);

  final AvroYarnJobSubmissionParameters jobSubmissionParameters =
      yarnClusterSubmissionFromCS.getYarnJobSubmissionParameters();
  jobSubmissionParameters.setDfsJobSubmissionFolder(jobFolderOnDFSPath);
  final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(jobSubmissionParameters.getSchema(),
      outputStream);
  datumWriter.write(jobSubmissionParameters, encoder);
  encoder.flush();
  outputStream.flush();
}
 

public static void writeToAvro(File inputFile, OutputStream outputStream)
    throws IOException {

  DataFileWriter<Stock> writer =
      new DataFileWriter<Stock>(
          new SpecificDatumWriter<Stock>());

  writer.setCodec(CodecFactory.snappyCodec());
  writer.create(Stock.SCHEMA$, outputStream);

  for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) {
    writer.append(stock);
  }

  IOUtils.closeStream(writer);
  IOUtils.closeStream(outputStream);
}
 

/**
 * generates the md5 hash of the schemaId and appends it to the given byte array.
 * the byte array representing the payload of a BrooklinEnvelope
 *
 * This is done so when the client decodes the payload, it will contain a schemaId which
 * can be used to retrieve the schema from the Schema Registry
 *
 * This method also converts an IndexedRecord into a byte array first
 */
public static byte[] encode(String schemaId, IndexedRecord record) throws AvroEncodingException {
  Validate.notNull(record, "cannot encode null Record, schemaId: " + schemaId);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  out.write(MAGIC_BYTE);
  byte[] md5Bytes = hexToMd5(schemaId);

  try {
    out.write(md5Bytes);
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<org.apache.avro.generic.IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(record.getSchema());
    } else {
      writer = new GenericDatumWriter<>(record.getSchema());
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (IOException e) {
    throw new AvroEncodingException(e);
  }

  return out.toByteArray();
}
 

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 

private byte[] serializeEvent(Event event, boolean useAvroEventFormat) throws IOException {
    byte[] bytes;
    if (useAvroEventFormat) {
        if (!tempOutStream.isPresent()) {
            tempOutStream = Optional.of(new ByteArrayOutputStream());
        }
        if (!writer.isPresent()) {
            writer = Optional.of(new SpecificDatumWriter<AvroFlumeEvent>(AvroFlumeEvent.class));
        }
        tempOutStream.get().reset();
        AvroFlumeEvent e = new AvroFlumeEvent(toCharSeqMap(event.getHeaders()),
                ByteBuffer.wrap(event.getBody()));
        encoder = EncoderFactory.get().directBinaryEncoder(tempOutStream.get(), encoder);
        writer.get().write(e, encoder);
        encoder.flush();
        bytes = tempOutStream.get().toByteArray();
    } else {
        bytes = event.getBody();
    }
    return bytes;
}
 

/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 

@Test
public void testAvroParquetReadWriteRoundTripWithoutSchemaRegistry() throws Exception {
    Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/unittest.avsc"));

    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    msg1 = builder.set("field1", "foo").set("field2", 1467176315L).build();
    msg2 = builder.set("field1", "bar").set("field2", 1467176344L).build();

    writer = new SpecificDatumWriter(schema);

    Map<String, String> avroSchemas = Stream
            .of(new String[][]{
                    {"test-avro-topic", "/unittest.avsc"}
            }).collect(Collectors.toMap(data -> data[0], data -> data[1]));
    when(config.getAvroMessageSchema()).thenReturn(avroSchemas);
    ConfigurableAvroSchemaRegistry configurableAvroSchemaRegistry = new ConfigurableAvroSchemaRegistry(config);
    mFactory.schemaRegistry = configurableAvroSchemaRegistry;
    when(config.getFileReaderWriterFactory())
            .thenReturn(AvroParquetFileReaderWriterFactory.class.getName());

    testAvroParquetReadWriteRoundTrip(configurableAvroSchemaRegistry);
}
 

@SuppressWarnings("unchecked")
public static synchronized <T> void sendMessageToKafka(T message, Class<T> clazz) {
    try {
        DatumWriter datumWriter;
        if (datumWriterPool.containsKey(clazz)) datumWriter = datumWriterPool.get(clazz);
        else {
            datumWriter = new SpecificDatumWriter<>(clazz);
            datumWriterPool.put(clazz, datumWriter);
        }
        datumWriter.write(message, encoder);
        encoder.flush();
        avroProducer.product(os.toByteArray());
        os.reset();
    } catch (Exception e) {
        _log.error(errorInfo(e));
    }
}
 

public byte[] serealizeAvroHttpRequestBinary(AvroHttpRequest request) {
    DatumWriter<AvroHttpRequest> writer = new SpecificDatumWriter<>(AvroHttpRequest.class);
    byte[] data = new byte[0];
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    Encoder jsonEncoder = EncoderFactory.get()
        .binaryEncoder(stream, null);
    try {
        writer.write(request, jsonEncoder);
        jsonEncoder.flush();
        data = stream.toByteArray();
    } catch (IOException e) {
        logger.error("Serialization error " + e.getMessage());
    }

    return data;
}
 

private byte[] getInsertMessageBytes(InsertMessage insertMessage){

        byte[] result = null;

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

        try {
            writer.write(insertMessage, encoder);
            encoder.flush();
            out.close();
            result = out.toByteArray();
        }catch (IOException e){
            return null;
        }

        return result;

    }
 

public byte[] toBytes(MeasurementV1 measurement) {
    try {
        ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
        byteStream.write(MAGIC_BYTE);
        byteStream.write(CURRENT_VERSION);
        BinaryEncoder encoder = encoderFactory.directBinaryEncoder(byteStream, null);
        DatumWriter<MeasurementV1> writer = new SpecificDatumWriter<MeasurementV1>(measurement.getSchema());
        writer.write(measurement, encoder);
        encoder.flush();
        byte[] result = byteStream.toByteArray();
        byteStream.close();
        return result;
    } catch (IOException e) {
        throw new RuntimeException("Error serializing Measurement object", e);
    }
}
 

/**
 * Serializes MultiRuntimeDefinition.
 * @param runtimeDefinition the Avro object to toString
 * @return Serialized avro string
 */
public String toString(final AvroMultiRuntimeDefinition runtimeDefinition){
  final DatumWriter<AvroMultiRuntimeDefinition> configurationWriter =
          new SpecificDatumWriter<>(AvroMultiRuntimeDefinition.class);
  final String serializedConfiguration;
  try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(runtimeDefinition.getSchema(), out);
    configurationWriter.write(runtimeDefinition, encoder);
    encoder.flush();
    out.flush();
    serializedConfiguration = out.toString(CHARSET_NAME);
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }

  return serializedConfiguration;
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);

	DatumWriter<E> datumWriter;
	Schema schema;
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
		datumWriter = new SpecificDatumWriter<E>(avroValueType);
		try {
			schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
		} catch (InstantiationException | IllegalAccessException e) {
			throw new RuntimeException(e.getMessage());
		}
	} else if (org.apache.avro.generic.GenericRecord.class.isAssignableFrom(avroValueType)) {
		if (userDefinedSchema == null) {
			throw new IllegalStateException("Schema must be set when using Generic Record");
		}
		datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
		schema = userDefinedSchema;
	} else {
		datumWriter = new ReflectDatumWriter<E>(avroValueType);
		schema = ReflectData.get().getSchema(avroValueType);
	}
	dataFileWriter = new DataFileWriter<E>(datumWriter);
	if (codec != null) {
		dataFileWriter.setCodec(codec.getCodecFactory());
	}
	if (userDefinedSchema == null) {
		dataFileWriter.create(schema, stream);
	} else {
		dataFileWriter.create(userDefinedSchema, stream);
	}
}
 

private void writeObject(java.io.ObjectOutputStream out)
        throws IOException {
    DatumWriter<LogLine> writer = new SpecificDatumWriter<LogLine>
            (LogLine.class);
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    writer.write(this, encoder);
    encoder.flush();
}
 

@Override
public void toFile(final Configuration conf, final File file) throws IOException {
  final AvroConfiguration avroConfiguration = toAvro(conf);
  final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class);
  try (DataFileWriter<AvroConfiguration> dataFileWriter = new DataFileWriter<>(configurationWriter)) {
    dataFileWriter.create(avroConfiguration.getSchema(), file);
    dataFileWriter.append(avroConfiguration);
  }
}
 

@Override
public void open(InitializationContext context) throws Exception {
	this.schema = AvroSchemaConverter.convertToSchema(rowType);
	datumWriter = new SpecificDatumWriter<>(schema);
	arrayOutputStream = new ByteArrayOutputStream();
	encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 

/**
 * Creates an {@link AvroWriterFactory} for an Avro specific type. The Avro writers
 * will use the schema of that specific type to build and write the records.
 *
 * @param type The class of the type to write.
 */
public static <T extends SpecificRecordBase> AvroWriterFactory<T> forSpecificRecord(Class<T> type) {
	String schemaString = SpecificData.get().getSchema(type).toString();
	AvroBuilder<T> builder = (out) -> createAvroDataFileWriter(
		schemaString,
		SpecificDatumWriter::new,
		out);
	return new AvroWriterFactory<>(builder);
}
 

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) {
	SpecificData specificData = new SpecificData(cl);
	Schema newSchema = extractAvroSpecificSchema(type, specificData);

	return new AvroFactory<>(
		specificData,
		newSchema,
		new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData),
		new SpecificDatumWriter<>(newSchema, specificData)
	);
}
 

static void writeAvroYarnAppSubmissionParametersToOutputStream(
    final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS,
    final OutputStream outputStream) throws IOException {
  final DatumWriter<AvroYarnAppSubmissionParameters> datumWriter =
      new SpecificDatumWriter<>(AvroYarnAppSubmissionParameters.class);

  final AvroYarnAppSubmissionParameters appSubmissionParameters =
      yarnClusterSubmissionFromCS.getYarnAppSubmissionParameters();
  final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(appSubmissionParameters.getSchema(), outputStream);
  datumWriter.write(appSubmissionParameters, encoder);
  encoder.flush();
  outputStream.flush();
}
 

private byte[] encodeRecord(IndexedRecord record) throws Exception {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  EncoderFactory encoderfactory = new EncoderFactory();
  BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
  DatumWriter<IndexedRecord> writer;
  if (record instanceof SpecificRecord) {
    writer = new SpecificDatumWriter<>(schema);
  } else {
    writer = new GenericDatumWriter<>(schema);
  }
  writer.write(record, encoder);
  encoder.flush(); //encoder may buffer
  return out.toByteArray();
}