Java Code Examples for org.apache.avro.io.DatumReader#setSchema()

The following examples show how to use org.apache.avro.io.DatumReader#setSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvroFileInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns a input stream given a file path
 *
 * @param path
 * @return InputStream
 * @throws IOException
 */
@Override
protected InputStream openFile(Path path) throws IOException
{
  InputStream is = super.openFile(path);
  if (is != null) {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
    avroDataStream = new DataFileStream<GenericRecord>(is, datumReader);
    datumReader.setSchema(avroDataStream.getSchema());
  }
  return is;
}
Example 2
Source File: TestFlumeFailoverTarget.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
@Test
public void testWriteAvroRecordsDropSchema() throws InterruptedException, StageException, IOException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.avroSchema = SdcAvroTestUtil.AVRO_SCHEMA1;
  dataGeneratorFormatConfig.avroSchemaSource = INLINE;
  dataGeneratorFormatConfig.includeSchema = false;
  dataGeneratorFormatConfig.avroCompression = AvroCompression.NULL;
  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createDefaultFlumeConfig(port, false),
    DataFormat.AVRO,
    dataGeneratorFormatConfig
  );
  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> records = SdcAvroTestUtil.getRecords1();
  targetRunner.runWrite(records);
  targetRunner.runDestroy();

  List<GenericRecord> genericRecords = new ArrayList<>();
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(); //Reader schema argument is optional
  datumReader.setSchema(new Schema.Parser().parse(SdcAvroTestUtil.AVRO_SCHEMA1));

  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  while(event != null) {
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
    GenericRecord read = datumReader.read(null, decoder);
    genericRecords.add(read);
    event = ch.take();
  }
  transaction.commit();
  transaction.close();

  Assert.assertEquals(3, genericRecords.size());
  SdcAvroTestUtil.compare1(genericRecords);
}
Example 3
Source File: ParquetReaderTest.java    From reef with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDataEntries() throws IOException, InjectionException {
  final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder();
  builder.bindNamedParameter(PathString.class, file.getAbsolutePath());
  final Configuration conf = builder.build();
  final Injector injector = Tang.Factory.getTang().newInjector(conf);

  final ParquetReader reader = injector.getInstance(ParquetReader.class);

  final byte[] byteArr = reader.serializeToByteBuffer().array();
  final ByteArrayInputStream inputStream = new ByteArrayInputStream(byteArr);
  final DatumReader datumReader = new GenericDatumReader<GenericRecord>();
  datumReader.setSchema(reader.createAvroSchema());

  final AvroKeyDeserializer deserializer
          = new AvroKeyDeserializer<GenericRecord>(reader.createAvroSchema(), reader.createAvroSchema(), datumReader);
  deserializer.open(inputStream);

  AvroWrapper<GenericRecord> record = null;

  for (int i = 0; i < 10; i = i + 1) {
    record = deserializer.deserialize(record);
    Assert.assertEquals("User_" + i, record.datum().get("name").toString());
    Assert.assertEquals(i, record.datum().get("age"));
    Assert.assertEquals("blue", record.datum().get("favorite_color").toString());
  }
}
Example 4
Source File: ProjectionDatumReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
private DatumReader<D> newDatumReader() {
  DatumReader<D> reader = (DatumReader<D>) getReader.apply(readSchema);
  reader.setSchema(fileSchema);
  return reader;
}
Example 5
Source File: AvroUtils.java    From java-11-examples with Apache License 2.0 4 votes vote down vote up 
public static Employee deserializeEmployee(byte[] data) throws IOException {
    DatumReader<Employee> employeeDatumReader = new SpecificDatumReader<>(Employee.class);
    BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null);
    employeeDatumReader.setSchema(Employee.getClassSchema());
    return employeeDatumReader.read(null, binaryDecoder);
}
Example 6
Source File: ProjectionDatumReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
private DatumReader<D> newDatumReader() {
  DatumReader<D> reader = (DatumReader<D>) getReader.apply(readSchema);
  reader.setSchema(fileSchema);
  return reader;
}
Example 7
Source File: TestFlumeFailoverTarget.java    From datacollector with Apache License 2.0 4 votes vote down vote up 
@Test
public void testWriteAvroRecordsDropSchemaSingleEvent() throws InterruptedException, StageException, IOException {

  DataGeneratorFormatConfig dataGeneratorFormatConfig = new DataGeneratorFormatConfig();
  dataGeneratorFormatConfig.avroSchema = SdcAvroTestUtil.AVRO_SCHEMA1;
  dataGeneratorFormatConfig.avroSchemaSource = INLINE;
  dataGeneratorFormatConfig.includeSchema = false;
  dataGeneratorFormatConfig.avroCompression = AvroCompression.NULL;
  FlumeTarget flumeTarget = FlumeTestUtil.createFlumeTarget(
    FlumeTestUtil.createDefaultFlumeConfig(port, true),
    DataFormat.AVRO,
    dataGeneratorFormatConfig
  );
  TargetRunner targetRunner = new TargetRunner.Builder(FlumeDTarget.class, flumeTarget).build();

  targetRunner.runInit();
  List<Record> records = SdcAvroTestUtil.getRecords1();
  targetRunner.runWrite(records);
  targetRunner.runDestroy();

  List<GenericRecord> genericRecords = new ArrayList<>();
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(); //Reader schema argument is optional
  datumReader.setSchema(new Schema.Parser().parse(SdcAvroTestUtil.AVRO_SCHEMA1));

  int eventCounter = 0;
  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  while(event != null) {
    eventCounter++;
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
    GenericRecord read = datumReader.read(null, decoder);
    while(read != null) {
      genericRecords.add(read);
      try {
        read = datumReader.read(null, decoder);
      } catch (EOFException e) {
        break;
      }
    }
    event = ch.take();
  }
  transaction.commit();
  transaction.close();

  Assert.assertEquals(1, eventCounter);
  Assert.assertEquals(3, genericRecords.size());
  SdcAvroTestUtil.compare1(genericRecords);
}