org.apache.avro.specific.SpecificDatumWriter Java Examples
The following examples show how to use
org.apache.avro.specific.SpecificDatumWriter.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AzureBlobAvroWriter.java From samza with Apache License 2.0 | 7 votes |
@VisibleForTesting byte[] encodeRecord(IndexedRecord record) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Schema schema = record.getSchema(); try { EncoderFactory encoderfactory = new EncoderFactory(); BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null); DatumWriter<IndexedRecord> writer; if (record instanceof SpecificRecord) { writer = new SpecificDatumWriter<>(schema); } else { writer = new GenericDatumWriter<>(schema); } writer.write(record, encoder); encoder.flush(); //encoder may buffer } catch (Exception e) { throw new SamzaException("Unable to serialize Avro record using schema within the record: " + schema.toString(), e); } return out.toByteArray(); }
Example #2
Source File: AvroLWM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License | 6 votes |
/** * 데이타 전송 * @param event * @throws Exception * @return void */ public void send(COL_LWM2M event) throws Exception { EncoderFactory avroEncoderFactory = EncoderFactory.get(); SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$); ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null); try { avroEventWriter.write(event, binaryEncoder); binaryEncoder.flush(); } catch (IOException e) { e.printStackTrace(); throw e; } IOUtils.closeQuietly(stream); KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>( TOPIC, stream.toByteArray()); producer.send(data); }
Example #3
Source File: MesosRemoteManagerCodec.java From reef with Apache License 2.0 | 6 votes |
@Override public byte[] encode(final EvaluatorControl evaluatorControl) { try { LOG.log(Level.FINEST, "Before Encoding: {0}", evaluatorControl.toString()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); final DatumWriter<EvaluatorControl> writer = new SpecificDatumWriter<>(EvaluatorControl.getClassSchema()); writer.write(evaluatorControl, encoder); encoder.flush(); out.close(); LOG.log(Level.FINEST, "After Encoding"); return out.toByteArray(); } catch (final IOException ex) { throw new RemoteRuntimeException(ex); } }
Example #4
Source File: AvroOneM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License | 6 votes |
/** * 데이타 전송 * @param event * @throws Exception * @return void */ public void send(COL_ONEM2M event) throws Exception { EncoderFactory avroEncoderFactory = EncoderFactory.get(); SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$); ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null); try { avroEventWriter.write(event, binaryEncoder); binaryEncoder.flush(); } catch (IOException e) { e.printStackTrace(); throw e; } IOUtils.closeQuietly(stream); KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>( TOPIC, stream.toByteArray()); producer.send(data); }
Example #5
Source File: DataSerializer.java From jMetalSP with MIT License | 6 votes |
public byte[] serializeMessage(S clazz, String path) { byte[] result = null; try { File file = new File(path); Schema schema = new Schema.Parser().parse(file); ByteArrayOutputStream out = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); DatumWriter<S> dataFileWriter = new SpecificDatumWriter<S>(schema); dataFileWriter.write(clazz, encoder); encoder.flush(); result=out.toByteArray(); out.close(); }catch (Exception ex){ ex.printStackTrace(); } return result; }
Example #6
Source File: FsSpecProducer.java From incubator-gobblin with Apache License 2.0 | 6 votes |
private void writeAvroJobSpec(AvroJobSpec jobSpec) throws IOException { DatumWriter<AvroJobSpec> datumWriter = new SpecificDatumWriter<>(AvroJobSpec.SCHEMA$); DataFileWriter<AvroJobSpec> dataFileWriter = new DataFileWriter<>(datumWriter); Path jobSpecPath = new Path(this.specConsumerPath, jobSpec.getUri()); //Write the new JobSpec to a temporary path first. Path tmpDir = new Path(this.specConsumerPath, "_tmp"); if (!fs.exists(tmpDir)) { fs.mkdirs(tmpDir); } Path tmpJobSpecPath = new Path(tmpDir, jobSpec.getUri()); OutputStream out = fs.create(tmpJobSpecPath); dataFileWriter.create(AvroJobSpec.SCHEMA$, out); dataFileWriter.append(jobSpec); dataFileWriter.close(); //Rename the JobSpec from temporary to final location. HadoopUtils.renamePath(fs, tmpJobSpecPath, jobSpecPath, true); }
Example #7
Source File: AvroSerializationSchema.java From flink with Apache License 2.0 | 6 votes |
protected void checkAvroInitialized() { if (datumWriter != null) { return; } ClassLoader cl = Thread.currentThread().getContextClassLoader(); if (SpecificRecord.class.isAssignableFrom(recordClazz)) { Schema schema = SpecificData.get().getSchema(recordClazz); this.datumWriter = new SpecificDatumWriter<>(schema); this.schema = schema; } else { this.schema = new Schema.Parser().parse(this.schemaString); GenericData genericData = new GenericData(cl); this.datumWriter = new GenericDatumWriter<>(schema, genericData); } this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().directBinaryEncoder(arrayOutputStream, null); }
Example #8
Source File: AvroSerializationSchema.java From pulsar with Apache License 2.0 | 6 votes |
@Override public byte[] serialize(T t) { if (null == t) { return null; } // Writer to serialize Avro record into a byte array. DatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema()); // Output stream to serialize records into byte array. ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream(); // Low-level class for serialization of Avro values. Encoder encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); arrayOutputStream.reset(); try { writer.write(t, encoder); encoder.flush(); } catch (IOException e) { throw new RuntimeException("Error while serializing the record to Avro", e); } return arrayOutputStream.toByteArray(); }
Example #9
Source File: TypeUtils.java From geowave with Apache License 2.0 | 6 votes |
private static <T> byte[] deserialize( final T avroObject, final Schema avroSchema, final Class<T> avroClass) throws IOException { final ByteArrayOutputStream os = new ByteArrayOutputStream(); final BinaryEncoder encoder = ef.binaryEncoder(os, null); if (!writers.containsKey(avroClass.toString())) { writers.put(avroClass.toString(), new SpecificDatumWriter<T>(avroSchema)); } final SpecificDatumWriter<T> writer = writers.get(avroClass.toString()); writer.write(avroObject, encoder); encoder.flush(); return os.toByteArray(); }
Example #10
Source File: AvroConfigurationSerializer.java From reef with Apache License 2.0 | 6 votes |
/** * Produce a JSON string that represents given configuration. * @param configuration Tang configuration to convert into a JSON string. * @param prettyPrint If true, use new lines and spaces to pretty print the JSON string. * If false (by default), output JSON as a single line. * @return A JSON string that corresponds to the given Tang configuration. */ public String toString(final Configuration configuration, final boolean prettyPrint) { final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class); final String result; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { // TODO [REEF-1536] Re-enable pretty printing when Avro 1.7.5 available on all environments: final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(AvroConfiguration.SCHEMA$, out); //, prettyPrint); configurationWriter.write(toAvro(configuration), encoder); encoder.flush(); out.flush(); result = out.toString(JSON_CHARSET); } catch (final IOException e) { throw new RuntimeException(e); } return result; }
Example #11
Source File: YarnSubmissionParametersFileGenerator.java From reef with Apache License 2.0 | 6 votes |
static void writeAvroYarnJobSubmissionParametersToOutputStream( final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS, final String jobFolderOnDFSPath, final OutputStream outputStream) throws IOException { final DatumWriter<AvroYarnJobSubmissionParameters> datumWriter = new SpecificDatumWriter<>(AvroYarnJobSubmissionParameters.class); final AvroYarnJobSubmissionParameters jobSubmissionParameters = yarnClusterSubmissionFromCS.getYarnJobSubmissionParameters(); jobSubmissionParameters.setDfsJobSubmissionFolder(jobFolderOnDFSPath); final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(jobSubmissionParameters.getSchema(), outputStream); datumWriter.write(jobSubmissionParameters, encoder); encoder.flush(); outputStream.flush(); }
Example #12
Source File: AvroStockFileWrite.java From hiped2 with Apache License 2.0 | 6 votes |
public static void writeToAvro(File inputFile, OutputStream outputStream) throws IOException { DataFileWriter<Stock> writer = new DataFileWriter<Stock>( new SpecificDatumWriter<Stock>()); writer.setCodec(CodecFactory.snappyCodec()); writer.create(Stock.SCHEMA$, outputStream); for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) { writer.append(stock); } IOUtils.closeStream(writer); IOUtils.closeStream(outputStream); }
Example #13
Source File: AvroMessageEncoderUtil.java From brooklin with BSD 2-Clause "Simplified" License | 6 votes |
/** * generates the md5 hash of the schemaId and appends it to the given byte array. * the byte array representing the payload of a BrooklinEnvelope * * This is done so when the client decodes the payload, it will contain a schemaId which * can be used to retrieve the schema from the Schema Registry * * This method also converts an IndexedRecord into a byte array first */ public static byte[] encode(String schemaId, IndexedRecord record) throws AvroEncodingException { Validate.notNull(record, "cannot encode null Record, schemaId: " + schemaId); ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(MAGIC_BYTE); byte[] md5Bytes = hexToMd5(schemaId); try { out.write(md5Bytes); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); DatumWriter<org.apache.avro.generic.IndexedRecord> writer; if (record instanceof SpecificRecord) { writer = new SpecificDatumWriter<>(record.getSchema()); } else { writer = new GenericDatumWriter<>(record.getSchema()); } writer.write(record, encoder); encoder.flush(); //encoder may buffer } catch (IOException e) { throw new AvroEncodingException(e); } return out.toByteArray(); }
Example #14
Source File: AvroLWM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License | 6 votes |
/** * 데이타 전송 * @param event * @throws Exception * @return void */ public void send(COL_LWM2M event) throws Exception { EncoderFactory avroEncoderFactory = EncoderFactory.get(); SpecificDatumWriter<COL_LWM2M> avroEventWriter = new SpecificDatumWriter<COL_LWM2M>(COL_LWM2M.SCHEMA$); ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null); try { avroEventWriter.write(event, binaryEncoder); binaryEncoder.flush(); } catch (IOException e) { e.printStackTrace(); throw e; } IOUtils.closeQuietly(stream); KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>( TOPIC, stream.toByteArray()); producer.send(data); }
Example #15
Source File: PulsarSink.java From pulsar-flume-ng-sink with Apache License 2.0 | 6 votes |
private byte[] serializeEvent(Event event, boolean useAvroEventFormat) throws IOException { byte[] bytes; if (useAvroEventFormat) { if (!tempOutStream.isPresent()) { tempOutStream = Optional.of(new ByteArrayOutputStream()); } if (!writer.isPresent()) { writer = Optional.of(new SpecificDatumWriter<AvroFlumeEvent>(AvroFlumeEvent.class)); } tempOutStream.get().reset(); AvroFlumeEvent e = new AvroFlumeEvent(toCharSeqMap(event.getHeaders()), ByteBuffer.wrap(event.getBody())); encoder = EncoderFactory.get().directBinaryEncoder(tempOutStream.get(), encoder); writer.get().write(e, encoder); encoder.flush(); bytes = tempOutStream.get().toByteArray(); } else { bytes = event.getBody(); } return bytes; }
Example #16
Source File: AvroOneM2MDataPublish.java From SDA with BSD 2-Clause "Simplified" License | 6 votes |
/** * 데이타 전송 * @param event * @throws Exception * @return void */ public void send(COL_ONEM2M event) throws Exception { EncoderFactory avroEncoderFactory = EncoderFactory.get(); SpecificDatumWriter<COL_ONEM2M> avroEventWriter = new SpecificDatumWriter<COL_ONEM2M>(COL_ONEM2M.SCHEMA$); ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream,null); try { avroEventWriter.write(event, binaryEncoder); binaryEncoder.flush(); } catch (IOException e) { e.printStackTrace(); throw e; } IOUtils.closeQuietly(stream); KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>( TOPIC, stream.toByteArray()); producer.send(data); }
Example #17
Source File: AvroParquetFileReaderWriterFactoryTest.java From secor with Apache License 2.0 | 6 votes |
@Test public void testAvroParquetReadWriteRoundTripWithoutSchemaRegistry() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/unittest.avsc")); GenericRecordBuilder builder = new GenericRecordBuilder(schema); msg1 = builder.set("field1", "foo").set("field2", 1467176315L).build(); msg2 = builder.set("field1", "bar").set("field2", 1467176344L).build(); writer = new SpecificDatumWriter(schema); Map<String, String> avroSchemas = Stream .of(new String[][]{ {"test-avro-topic", "/unittest.avsc"} }).collect(Collectors.toMap(data -> data[0], data -> data[1])); when(config.getAvroMessageSchema()).thenReturn(avroSchemas); ConfigurableAvroSchemaRegistry configurableAvroSchemaRegistry = new ConfigurableAvroSchemaRegistry(config); mFactory.schemaRegistry = configurableAvroSchemaRegistry; when(config.getFileReaderWriterFactory()) .thenReturn(AvroParquetFileReaderWriterFactory.class.getName()); testAvroParquetReadWriteRoundTrip(configurableAvroSchemaRegistry); }
Example #18
Source File: KafkaUtils.java From yuzhouwan with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") public static synchronized <T> void sendMessageToKafka(T message, Class<T> clazz) { try { DatumWriter datumWriter; if (datumWriterPool.containsKey(clazz)) datumWriter = datumWriterPool.get(clazz); else { datumWriter = new SpecificDatumWriter<>(clazz); datumWriterPool.put(clazz, datumWriter); } datumWriter.write(message, encoder); encoder.flush(); avroProducer.product(os.toByteArray()); os.reset(); } catch (Exception e) { _log.error(errorInfo(e)); } }
Example #19
Source File: AvroSerealizer.java From tutorials with MIT License | 6 votes |
public byte[] serealizeAvroHttpRequestBinary(AvroHttpRequest request) { DatumWriter<AvroHttpRequest> writer = new SpecificDatumWriter<>(AvroHttpRequest.class); byte[] data = new byte[0]; ByteArrayOutputStream stream = new ByteArrayOutputStream(); Encoder jsonEncoder = EncoderFactory.get() .binaryEncoder(stream, null); try { writer.write(request, jsonEncoder); jsonEncoder.flush(); data = stream.toByteArray(); } catch (IOException e) { logger.error("Serialization error " + e.getMessage()); } return data; }
Example #20
Source File: JavaToAvroSerializer.java From Decision with Apache License 2.0 | 6 votes |
private byte[] getInsertMessageBytes(InsertMessage insertMessage){ byte[] result = null; ByteArrayOutputStream out = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); SpecificDatumWriter writer = new SpecificDatumWriter<InsertMessage>(InsertMessage.getClassSchema()); try { writer.write(insertMessage, encoder); encoder.flush(); out.close(); result = out.toByteArray(); }catch (IOException e){ return null; } return result; }
Example #21
Source File: InternalAvroSerde.java From kafka-metrics with Apache License 2.0 | 6 votes |
public byte[] toBytes(MeasurementV1 measurement) { try { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); byteStream.write(MAGIC_BYTE); byteStream.write(CURRENT_VERSION); BinaryEncoder encoder = encoderFactory.directBinaryEncoder(byteStream, null); DatumWriter<MeasurementV1> writer = new SpecificDatumWriter<MeasurementV1>(measurement.getSchema()); writer.write(measurement, encoder); encoder.flush(); byte[] result = byteStream.toByteArray(); byteStream.close(); return result; } catch (IOException e) { throw new RuntimeException("Error serializing Measurement object", e); } }
Example #22
Source File: MultiRuntimeDefinitionSerializer.java From reef with Apache License 2.0 | 6 votes |
/** * Serializes MultiRuntimeDefinition. * @param runtimeDefinition the Avro object to toString * @return Serialized avro string */ public String toString(final AvroMultiRuntimeDefinition runtimeDefinition){ final DatumWriter<AvroMultiRuntimeDefinition> configurationWriter = new SpecificDatumWriter<>(AvroMultiRuntimeDefinition.class); final String serializedConfiguration; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(runtimeDefinition.getSchema(), out); configurationWriter.write(runtimeDefinition, encoder); encoder.flush(); out.flush(); serializedConfiguration = out.toString(CHARSET_NAME); } catch (final IOException e) { throw new RuntimeException(e); } return serializedConfiguration; }
Example #23
Source File: AvroOutputFormat.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(int taskNumber, int numTasks) throws IOException { super.open(taskNumber, numTasks); DatumWriter<E> datumWriter; Schema schema; if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) { datumWriter = new SpecificDatumWriter<E>(avroValueType); try { schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException(e.getMessage()); } } else if (org.apache.avro.generic.GenericRecord.class.isAssignableFrom(avroValueType)) { if (userDefinedSchema == null) { throw new IllegalStateException("Schema must be set when using Generic Record"); } datumWriter = new GenericDatumWriter<E>(userDefinedSchema); schema = userDefinedSchema; } else { datumWriter = new ReflectDatumWriter<E>(avroValueType); schema = ReflectData.get().getSchema(avroValueType); } dataFileWriter = new DataFileWriter<E>(datumWriter); if (codec != null) { dataFileWriter.setCodec(codec.getCodecFactory()); } if (userDefinedSchema == null) { dataFileWriter.create(schema, stream); } else { dataFileWriter.create(userDefinedSchema, stream); } }
Example #24
Source File: JavaSessionize.java From hadoop-arch-book with Apache License 2.0 | 5 votes |
private void writeObject(java.io.ObjectOutputStream out) throws IOException { DatumWriter<LogLine> writer = new SpecificDatumWriter<LogLine> (LogLine.class); Encoder encoder = EncoderFactory.get().binaryEncoder(out, null); writer.write(this, encoder); encoder.flush(); }
Example #25
Source File: AvroConfigurationSerializer.java From reef with Apache License 2.0 | 5 votes |
@Override public void toFile(final Configuration conf, final File file) throws IOException { final AvroConfiguration avroConfiguration = toAvro(conf); final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class); try (DataFileWriter<AvroConfiguration> dataFileWriter = new DataFileWriter<>(configurationWriter)) { dataFileWriter.create(avroConfiguration.getSchema(), file); dataFileWriter.append(avroConfiguration); } }
Example #26
Source File: AvroRowDataSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(InitializationContext context) throws Exception { this.schema = AvroSchemaConverter.convertToSchema(rowType); datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
Example #27
Source File: AvroWriters.java From flink with Apache License 2.0 | 5 votes |
/** * Creates an {@link AvroWriterFactory} for an Avro specific type. The Avro writers * will use the schema of that specific type to build and write the records. * * @param type The class of the type to write. */ public static <T extends SpecificRecordBase> AvroWriterFactory<T> forSpecificRecord(Class<T> type) { String schemaString = SpecificData.get().getSchema(type).toString(); AvroBuilder<T> builder = (out) -> createAvroDataFileWriter( schemaString, SpecificDatumWriter::new, out); return new AvroWriterFactory<>(builder); }
Example #28
Source File: AvroFactory.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) { SpecificData specificData = new SpecificData(cl); Schema newSchema = extractAvroSpecificSchema(type, specificData); return new AvroFactory<>( specificData, newSchema, new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData), new SpecificDatumWriter<>(newSchema, specificData) ); }
Example #29
Source File: YarnSubmissionParametersFileGenerator.java From reef with Apache License 2.0 | 5 votes |
static void writeAvroYarnAppSubmissionParametersToOutputStream( final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS, final OutputStream outputStream) throws IOException { final DatumWriter<AvroYarnAppSubmissionParameters> datumWriter = new SpecificDatumWriter<>(AvroYarnAppSubmissionParameters.class); final AvroYarnAppSubmissionParameters appSubmissionParameters = yarnClusterSubmissionFromCS.getYarnAppSubmissionParameters(); final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(appSubmissionParameters.getSchema(), outputStream); datumWriter.write(appSubmissionParameters, encoder); encoder.flush(); outputStream.flush(); }
Example #30
Source File: TestAzureBlobAvroWriter.java From samza with Apache License 2.0 | 5 votes |
private byte[] encodeRecord(IndexedRecord record) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Schema schema = record.getSchema(); EncoderFactory encoderfactory = new EncoderFactory(); BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null); DatumWriter<IndexedRecord> writer; if (record instanceof SpecificRecord) { writer = new SpecificDatumWriter<>(schema); } else { writer = new GenericDatumWriter<>(schema); } writer.write(record, encoder); encoder.flush(); //encoder may buffer return out.toByteArray(); }