org.apache.avro.specific.SpecificDatumWriter Java Examples

The following examples show how to use org.apache.avro.specific.SpecificDatumWriter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AzureBlobAvroWriter.java    From samza with Apache License 2.0 7 votes vote down vote up
@VisibleForTesting
byte[] encodeRecord(IndexedRecord record) {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  try {
    EncoderFactory encoderfactory = new EncoderFactory();
    BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
    DatumWriter<IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(schema);
    } else {
      writer = new GenericDatumWriter<>(schema);
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (Exception e) {
    throw new SamzaException("Unable to serialize Avro record using schema within the record: " + schema.toString(), e);
  }
  return out.toByteArray();
}
 
Example #2
Source File: AvroLWM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #3
Source File: MesosRemoteManagerCodec.java    From reef with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] encode(final EvaluatorControl evaluatorControl) {
  try {
    LOG.log(Level.FINEST, "Before Encoding: {0}", evaluatorControl.toString());
    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    final DatumWriter<EvaluatorControl> writer = new SpecificDatumWriter<>(EvaluatorControl.getClassSchema());
    writer.write(evaluatorControl, encoder);
    encoder.flush();
    out.close();
    LOG.log(Level.FINEST, "After Encoding");
    return out.toByteArray();
  } catch (final IOException ex) {
    throw new RemoteRuntimeException(ex);
  }
}
 
Example #4
Source File: AvroOneM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #5
Source File: DataSerializer.java    From jMetalSP with MIT License 6 votes vote down vote up
public byte[] serializeMessage(S clazz, String path) {

        byte[] result = null;
        try {

            File file = new File(path);

            Schema schema = new Schema.Parser().parse(file);

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
            DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema);
            dataFileWriter.write(clazz, encoder);
            encoder.flush();
            result=out.toByteArray();
            out.close();
        }catch (Exception ex){
            ex.printStackTrace();
        }
        return result;
    }
 
Example #6
Source File: FsSpecProducer.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void writeAvroJobSpec(AvroJobSpec jobSpec) throws IOException {
  DatumWriter<AvroJobSpec> datumWriter = new SpecificDatumWriter<>(AvroJobSpec.SCHEMA$);
  DataFileWriter<AvroJobSpec> dataFileWriter = new DataFileWriter<>(datumWriter);

  Path jobSpecPath = new Path(this.specConsumerPath, jobSpec.getUri());

  //Write the new JobSpec to a temporary path first.
  Path tmpDir = new Path(this.specConsumerPath, "_tmp");
  if (!fs.exists(tmpDir)) {
    fs.mkdirs(tmpDir);
  }

  Path tmpJobSpecPath = new Path(tmpDir, jobSpec.getUri());

  OutputStream out = fs.create(tmpJobSpecPath);

  dataFileWriter.create(AvroJobSpec.SCHEMA$, out);
  dataFileWriter.append(jobSpec);
  dataFileWriter.close();

  //Rename the JobSpec from temporary to final location.
  HadoopUtils.renamePath(fs, tmpJobSpecPath, jobSpecPath, true);
}
 
Example #7
Source File: AvroSerializationSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
protected void checkAvroInitialized() {
	if (datumWriter != null) {
		return;
	}
	ClassLoader cl = Thread.currentThread().getContextClassLoader();
		if (SpecificRecord.class.isAssignableFrom(recordClazz)) {
		Schema schema = SpecificData.get().getSchema(recordClazz);
		this.datumWriter = new SpecificDatumWriter<>(schema);
		this.schema = schema;
	} else {
		this.schema = new Schema.Parser().parse(this.schemaString);
		GenericData genericData = new GenericData(cl);

		this.datumWriter = new GenericDatumWriter<>(schema, genericData);
	}
	this.arrayOutputStream = new ByteArrayOutputStream();
	this.encoder = EncoderFactory.get().directBinaryEncoder(arrayOutputStream, null);
}
 
Example #8
Source File: AvroSerializationSchema.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] serialize(T t) {
    if (null == t) {
        return null;
    }

    // Writer to serialize Avro record into a byte array.
    DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());
    // Output stream to serialize records into byte array.
    ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
    // Low-level class for serialization of Avro values.
    Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
    arrayOutputStream.reset();
    try {
        writer.write(t, encoder);
        encoder.flush();
    } catch (IOException e) {
        throw new RuntimeException("Error while serializing the record to Avro", e);
    }

    return arrayOutputStream.toByteArray();
}
 
Example #9
Source File: TypeUtils.java    From geowave with Apache License 2.0 6 votes vote down vote up
private static <T> byte[] deserialize(
    final T avroObject,
    final Schema avroSchema,
    final Class<T> avroClass) throws IOException {

  final ByteArrayOutputStream os = new ByteArrayOutputStream();
  final BinaryEncoder encoder = ef.binaryEncoder(os, null);
  if (!writers.containsKey(avroClass.toString())) {
    writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema));
  }

  final SpecificDatumWriter<T> writer = writers.get(avroClass.toString());
  writer.write(avroObject, encoder);
  encoder.flush();
  return os.toByteArray();
}
 
Example #10
Source File: AvroConfigurationSerializer.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Produce a JSON string that represents given configuration.
 * @param configuration Tang configuration to convert into a JSON string.
 * @param prettyPrint If true, use new lines and spaces to pretty print the JSON string.
 * If false (by default), output JSON as a single line.
 * @return A JSON string that corresponds to the given Tang configuration.
 */
public String toString(final Configuration configuration, final boolean prettyPrint) {
  final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class);
  final String result;
  try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
    // TODO [REEF-1536] Re-enable pretty printing when Avro 1.7.5 available on all environments:
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(AvroConfiguration.SCHEMA$, out); //, prettyPrint);
    configurationWriter.write(toAvro(configuration), encoder);
    encoder.flush();
    out.flush();
    result = out.toString(JSON_CHARSET);
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
  return result;
}
 
Example #11
Source File: YarnSubmissionParametersFileGenerator.java    From reef with Apache License 2.0 6 votes vote down vote up
static void writeAvroYarnJobSubmissionParametersToOutputStream(
    final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS,
    final String jobFolderOnDFSPath,
    final OutputStream outputStream) throws IOException {
  final DatumWriter<AvroYarnJobSubmissionParameters> datumWriter =
      new SpecificDatumWriter<>(AvroYarnJobSubmissionParameters.class);

  final AvroYarnJobSubmissionParameters jobSubmissionParameters =
      yarnClusterSubmissionFromCS.getYarnJobSubmissionParameters();
  jobSubmissionParameters.setDfsJobSubmissionFolder(jobFolderOnDFSPath);
  final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(jobSubmissionParameters.getSchema(),
      outputStream);
  datumWriter.write(jobSubmissionParameters, encoder);
  encoder.flush();
  outputStream.flush();
}
 
Example #12
Source File: AvroStockFileWrite.java    From hiped2 with Apache License 2.0 6 votes vote down vote up
public static void writeToAvro(File inputFile, OutputStream outputStream)
    throws IOException {

  DataFileWriter<Stock> writer =
      new DataFileWriter<Stock>(
          new SpecificDatumWriter<Stock>());

  writer.setCodec(CodecFactory.snappyCodec());
  writer.create(Stock.SCHEMA$, outputStream);

  for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) {
    writer.append(stock);
  }

  IOUtils.closeStream(writer);
  IOUtils.closeStream(outputStream);
}
 
Example #13
Source File: AvroMessageEncoderUtil.java    From brooklin with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * generates the md5 hash of the schemaId and appends it to the given byte array.
 * the byte array representing the payload of a BrooklinEnvelope
 *
 * This is done so when the client decodes the payload, it will contain a schemaId which
 * can be used to retrieve the schema from the Schema Registry
 *
 * This method also converts an IndexedRecord into a byte array first
 */
public static byte[] encode(String schemaId, IndexedRecord record) throws AvroEncodingException {
  Validate.notNull(record, "cannot encode null Record, schemaId: " + schemaId);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  out.write(MAGIC_BYTE);
  byte[] md5Bytes = hexToMd5(schemaId);

  try {
    out.write(md5Bytes);
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<org.apache.avro.generic.IndexedRecord> writer;
    if (record instanceof SpecificRecord) {
      writer = new SpecificDatumWriter<>(record.getSchema());
    } else {
      writer = new GenericDatumWriter<>(record.getSchema());
    }
    writer.write(record, encoder);
    encoder.flush(); //encoder may buffer
  } catch (IOException e) {
    throw new AvroEncodingException(e);
  }

  return out.toByteArray();
}
 
Example #14
Source File: AvroLWM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_LWM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #15
Source File: PulsarSink.java    From pulsar-flume-ng-sink with Apache License 2.0 6 votes vote down vote up
private byte[] serializeEvent(Event event, boolean useAvroEventFormat) throws IOException {
    byte[] bytes;
    if (useAvroEventFormat) {
        if (!tempOutStream.isPresent()) {
            tempOutStream = Optional.of(new ByteArrayOutputStream());
        }
        if (!writer.isPresent()) {
            writer = Optional.of(new SpecificDatumWriter<AvroFlumeEvent>(AvroFlumeEvent.class));
        }
        tempOutStream.get().reset();
        AvroFlumeEvent e = new AvroFlumeEvent(toCharSeqMap(event.getHeaders()),
                ByteBuffer.wrap(event.getBody()));
        encoder = EncoderFactory.get().directBinaryEncoder(tempOutStream.get(), encoder);
        writer.get().write(e, encoder);
        encoder.flush();
        bytes = tempOutStream.get().toByteArray();
    } else {
        bytes = event.getBody();
    }
    return bytes;
}
 
Example #16
Source File: AvroOneM2MDataPublish.java    From SDA with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * 데이타 전송
 * @param event
 * @throws Exception
 * @return void
 */
public void send(COL_ONEM2M event) throws Exception {
	EncoderFactory avroEncoderFactory = EncoderFactory.get();
	SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$);
	
	ByteArrayOutputStream stream = new ByteArrayOutputStream();
	BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null);

	try {
		avroEventWriter.write(event, binaryEncoder);
		binaryEncoder.flush();
	} catch (IOException e) {
		e.printStackTrace();
		throw e;
	}
	IOUtils.closeQuietly(stream);

	KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>(
			TOPIC, stream.toByteArray());

	producer.send(data);
}
 
Example #17
Source File: AvroParquetFileReaderWriterFactoryTest.java    From secor with Apache License 2.0 6 votes vote down vote up
@Test
public void testAvroParquetReadWriteRoundTripWithoutSchemaRegistry() throws Exception {
    Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/unittest.avsc"));

    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    msg1 = builder.set("field1", "foo").set("field2", 1467176315L).build();
    msg2 = builder.set("field1", "bar").set("field2", 1467176344L).build();

    writer = new SpecificDatumWriter(schema);

    Map<String, String> avroSchemas = Stream
            .of(new String[][]{
                    {"test-avro-topic", "/unittest.avsc"}
            }).collect(Collectors.toMap(data -> data[0], data -> data[1]));
    when(config.getAvroMessageSchema()).thenReturn(avroSchemas);
    ConfigurableAvroSchemaRegistry configurableAvroSchemaRegistry = new ConfigurableAvroSchemaRegistry(config);
    mFactory.schemaRegistry = configurableAvroSchemaRegistry;
    when(config.getFileReaderWriterFactory())
            .thenReturn(AvroParquetFileReaderWriterFactory.class.getName());

    testAvroParquetReadWriteRoundTrip(configurableAvroSchemaRegistry);
}
 
Example #18
Source File: KafkaUtils.java    From yuzhouwan with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static synchronized <T> void sendMessageToKafka(T message, Class<T> clazz) {
    try {
        DatumWriter datumWriter;
        if (datumWriterPool.containsKey(clazz)) datumWriter = datumWriterPool.get(clazz);
        else {
            datumWriter = new SpecificDatumWriter<>(clazz);
            datumWriterPool.put(clazz, datumWriter);
        }
        datumWriter.write(message, encoder);
        encoder.flush();
        avroProducer.product(os.toByteArray());
        os.reset();
    } catch (Exception e) {
        _log.error(errorInfo(e));
    }
}
 
Example #19
Source File: AvroSerealizer.java    From tutorials with MIT License 6 votes vote down vote up
public byte[] serealizeAvroHttpRequestBinary(AvroHttpRequest request) {
    DatumWriter<AvroHttpRequest> writer = new SpecificDatumWriter<>(AvroHttpRequest.class);
    byte[] data = new byte[0];
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    Encoder jsonEncoder = EncoderFactory.get()
        .binaryEncoder(stream, null);
    try {
        writer.write(request, jsonEncoder);
        jsonEncoder.flush();
        data = stream.toByteArray();
    } catch (IOException e) {
        logger.error("Serialization error " + e.getMessage());
    }

    return data;
}
 
Example #20
Source File: JavaToAvroSerializer.java    From Decision with Apache License 2.0 6 votes vote down vote up
private byte[] getInsertMessageBytes(InsertMessage insertMessage){

        byte[] result = null;

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
        SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema());

        try {
            writer.write(insertMessage, encoder);
            encoder.flush();
            out.close();
            result = out.toByteArray();
        }catch (IOException e){
            return null;
        }

        return result;

    }
 
Example #21
Source File: InternalAvroSerde.java    From kafka-metrics with Apache License 2.0 6 votes vote down vote up
public byte[] toBytes(MeasurementV1 measurement) {
    try {
        ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
        byteStream.write(MAGIC_BYTE);
        byteStream.write(CURRENT_VERSION);
        BinaryEncoder encoder = encoderFactory.directBinaryEncoder(byteStream, null);
        DatumWriter<MeasurementV1> writer = new SpecificDatumWriter<MeasurementV1>(measurement.getSchema());
        writer.write(measurement, encoder);
        encoder.flush();
        byte[] result = byteStream.toByteArray();
        byteStream.close();
        return result;
    } catch (IOException e) {
        throw new RuntimeException("Error serializing Measurement object", e);
    }
}
 
Example #22
Source File: MultiRuntimeDefinitionSerializer.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Serializes MultiRuntimeDefinition.
 * @param runtimeDefinition the Avro object to toString
 * @return Serialized avro string
 */
public String toString(final AvroMultiRuntimeDefinition runtimeDefinition){
  final DatumWriter<AvroMultiRuntimeDefinition> configurationWriter =
          new SpecificDatumWriter<>(AvroMultiRuntimeDefinition.class);
  final String serializedConfiguration;
  try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(runtimeDefinition.getSchema(), out);
    configurationWriter.write(runtimeDefinition, encoder);
    encoder.flush();
    out.flush();
    serializedConfiguration = out.toString(CHARSET_NAME);
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }

  return serializedConfiguration;
}
 
Example #23
Source File: AvroOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);

	DatumWriter<E> datumWriter;
	Schema schema;
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
		datumWriter = new SpecificDatumWriter<E>(avroValueType);
		try {
			schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
		} catch (InstantiationException | IllegalAccessException e) {
			throw new RuntimeException(e.getMessage());
		}
	} else if (org.apache.avro.generic.GenericRecord.class.isAssignableFrom(avroValueType)) {
		if (userDefinedSchema == null) {
			throw new IllegalStateException("Schema must be set when using Generic Record");
		}
		datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
		schema = userDefinedSchema;
	} else {
		datumWriter = new ReflectDatumWriter<E>(avroValueType);
		schema = ReflectData.get().getSchema(avroValueType);
	}
	dataFileWriter = new DataFileWriter<E>(datumWriter);
	if (codec != null) {
		dataFileWriter.setCodec(codec.getCodecFactory());
	}
	if (userDefinedSchema == null) {
		dataFileWriter.create(schema, stream);
	} else {
		dataFileWriter.create(userDefinedSchema, stream);
	}
}
 
Example #24
Source File: JavaSessionize.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
private void writeObject(java.io.ObjectOutputStream out)
        throws IOException {
    DatumWriter<LogLine> writer = new SpecificDatumWriter<LogLine>
            (LogLine.class);
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    writer.write(this, encoder);
    encoder.flush();
}
 
Example #25
Source File: AvroConfigurationSerializer.java    From reef with Apache License 2.0 5 votes vote down vote up
@Override
public void toFile(final Configuration conf, final File file) throws IOException {
  final AvroConfiguration avroConfiguration = toAvro(conf);
  final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class);
  try (DataFileWriter<AvroConfiguration> dataFileWriter = new DataFileWriter<>(configurationWriter)) {
    dataFileWriter.create(avroConfiguration.getSchema(), file);
    dataFileWriter.append(avroConfiguration);
  }
}
 
Example #26
Source File: AvroRowDataSerializationSchema.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(InitializationContext context) throws Exception {
	this.schema = AvroSchemaConverter.convertToSchema(rowType);
	datumWriter = new SpecificDatumWriter<>(schema);
	arrayOutputStream = new ByteArrayOutputStream();
	encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 
Example #27
Source File: AvroWriters.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an {@link AvroWriterFactory} for an Avro specific type. The Avro writers
 * will use the schema of that specific type to build and write the records.
 *
 * @param type The class of the type to write.
 */
public static <T extends SpecificRecordBase> AvroWriterFactory<T> forSpecificRecord(Class<T> type) {
	String schemaString = SpecificData.get().getSchema(type).toString();
	AvroBuilder<T> builder = (out) -> createAvroDataFileWriter(
		schemaString,
		SpecificDatumWriter::new,
		out);
	return new AvroWriterFactory<>(builder);
}
 
Example #28
Source File: AvroFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) {
	SpecificData specificData = new SpecificData(cl);
	Schema newSchema = extractAvroSpecificSchema(type, specificData);

	return new AvroFactory<>(
		specificData,
		newSchema,
		new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData),
		new SpecificDatumWriter<>(newSchema, specificData)
	);
}
 
Example #29
Source File: YarnSubmissionParametersFileGenerator.java    From reef with Apache License 2.0 5 votes vote down vote up
static void writeAvroYarnAppSubmissionParametersToOutputStream(
    final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS,
    final OutputStream outputStream) throws IOException {
  final DatumWriter<AvroYarnAppSubmissionParameters> datumWriter =
      new SpecificDatumWriter<>(AvroYarnAppSubmissionParameters.class);

  final AvroYarnAppSubmissionParameters appSubmissionParameters =
      yarnClusterSubmissionFromCS.getYarnAppSubmissionParameters();
  final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(appSubmissionParameters.getSchema(), outputStream);
  datumWriter.write(appSubmissionParameters, encoder);
  encoder.flush();
  outputStream.flush();
}
 
Example #30
Source File: TestAzureBlobAvroWriter.java    From samza with Apache License 2.0 5 votes vote down vote up
private byte[] encodeRecord(IndexedRecord record) throws Exception {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  EncoderFactory encoderfactory = new EncoderFactory();
  BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
  DatumWriter<IndexedRecord> writer;
  if (record instanceof SpecificRecord) {
    writer = new SpecificDatumWriter<>(schema);
  } else {
    writer = new GenericDatumWriter<>(schema);
  }
  writer.write(record, encoder);
  encoder.flush(); //encoder may buffer
  return out.toByteArray();
}