org.apache.avro.specific.SpecificData Java Examples

void checkAvroInitialized() {
	if (datumReader != null) {
		return;
	}

	ClassLoader cl = Thread.currentThread().getContextClassLoader();
	if (SpecificRecord.class.isAssignableFrom(recordClazz)) {
		SpecificData specificData = new SpecificData(cl);
		this.datumReader = new SpecificDatumReader<>(specificData);
		this.reader = specificData.getSchema(recordClazz);
	} else {
		this.reader = new Schema.Parser().parse(schemaString);
		GenericData genericData = new GenericData(cl);
		this.datumReader = new GenericDatumReader<>(null, this.reader, genericData);
	}

	this.inputStream = new MutableByteArrayInputStream();
	this.decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
}
 

void checkAvroInitialized() {
	if (datumReader != null) {
		return;
	}

	ClassLoader cl = Thread.currentThread().getContextClassLoader();
	if (SpecificRecord.class.isAssignableFrom(recordClazz)) {
		SpecificData specificData = new SpecificData(cl);
		this.datumReader = new SpecificDatumReader<>(specificData);
		this.reader = specificData.getSchema(recordClazz);
	} else {
		this.reader = new Schema.Parser().parse(schemaString);
		GenericData genericData = new GenericData(cl);
		this.datumReader = new GenericDatumReader<>(null, this.reader, genericData);
	}

	this.inputStream = new MutableByteArrayInputStream();
	this.decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
}
 

@Test
public void testReadUUIDList() throws IOException {
  Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  uuidListSchema.getField("uuids").schema().addProp(
      SpecificData.CLASS_PROP, List.class.getName());
  LogicalTypes.uuid().addToSchema(
      uuidListSchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidListSchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDList expected = new RecordWithUUIDList();
  expected.uuids = Arrays.asList(u1, u2);

  File test = write(uuidListSchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected, read(REFLECT, uuidListSchema, test).get(0));
}
 

/**
 * Converts an Avro class into a nested row structure with deterministic field order and data
 * types that are compatible with Flink's Table & SQL API.
 *
 * @param avroClass Avro specific record that contains schema information
 * @return type information matching the schema
 */
@SuppressWarnings("unchecked")
public static <T extends SpecificRecord> TypeInformation<Row> convertToTypeInfo(Class<T> avroClass) {
	Preconditions.checkNotNull(avroClass, "Avro specific record class must not be null.");
	// determine schema to retrieve deterministic field order
	final Schema schema = SpecificData.get().getSchema(avroClass);
	return (TypeInformation<Row>) convertToTypeInfo(schema);
}
 

@Override
@SuppressWarnings("unchecked")
public void start() {
  // Should do the right thing whether it is generic or specific
  this.currentRecord = (T) ((this.specificClass == null) ?
          new GenericData.Record(avroSchema) :
          SpecificData.newInstance(specificClass, avroSchema));
}
 

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException {
	recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject();
	schemaString = (String) inputStream.readObject();
	if (recordClazz != null) {
		schema = SpecificData.get().getSchema(recordClazz);
	} else {
		schema = new Schema.Parser().parse(schemaString);
	}
	datumWriter = new SpecificDatumWriter<>(schema);
	arrayOutputStream = new ByteArrayOutputStream();
	encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 

@Test
public void demonstrateDefaultValueBehaviour() throws Exception {
  Schema schema = by14.HasComplexDefaults.SCHEMA$;
  Schema.Field field = schema.getField("fieldWithDefaultRecord");

  Object specificDefault = SpecificData.get().getDefaultValue(field);
  Assert.assertNotNull(specificDefault);
  Assert.assertTrue(specificDefault instanceof by14.DefaultRecord);
  Object genericDefault = GenericData.get().getDefaultValue(field);
  Assert.assertNotNull(genericDefault);
  Assert.assertTrue(genericDefault instanceof GenericData.Record);
}
 

@Override
public D fromGeneric(final GenericRecord serialized) {
    GenericData.get().addLogicalTypeConversion(new Conversions.DecimalConversion());
    SpecificData specificData = SpecificData.get();
    specificData.addLogicalTypeConversion(new Conversions.DecimalConversion());
    return isNull(serialized) ? null : (D) specificData.deepCopy(serialized.getSchema(), serialized);
}
 

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) {
	SpecificData specificData = new SpecificData(cl);
	Schema newSchema = specificData.getSchema(type);

	return new AvroFactory<>(
		specificData,
		newSchema,
		new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData),
		new SpecificDatumWriter<>(newSchema, specificData)
	);
}
 

@Test
public void testWriteParquetAvroGeneric() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final Schema schema = Address.getClassSchema();

	final Collection<GenericRecord> data = new GenericTestDataCollection();

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<GenericRecord> stream = env.addSource(
			new FiniteTestSource<>(data), new GenericRecordAvroTypeInfo(schema));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forGenericRecord(schema))
					.build());

	env.execute();

	List<Address> expected = Arrays.asList(
			new Address(1, "a", "b", "c", "12345"),
			new Address(2, "x", "y", "z", "98765"));

	validateResults(folder, SpecificData.get(), expected);
}
 

private void _testSerDes(Long id, Number serdesProtocolVersion) throws Exception {
    SchemaMetadata schemaMetadata =
            new SchemaMetadata.Builder("random-" + System.currentTimeMillis())
                    .schemaGroup("custom")
                    .type(AvroSchemaProvider.TYPE)
                    .compatibility(SchemaCompatibility.BACKWARD)
                    .build();

    SchemaIdVersion schemaIdVersion = new SchemaIdVersion(1L, 1, id);

    Device input = new Device(1L, "device", 1, System.currentTimeMillis());
    SchemaVersionInfo schemaVersionInfo = new SchemaVersionInfo(id, input.getName().toString(), schemaIdVersion.getVersion(),
                                                                input.getSchema().toString(),
                                                                System.currentTimeMillis(),
                                                                "some device");

    new Expectations() {
        {
            mockSchemaRegistryClient.getSchemaMetadataInfo(anyString);
            result = new SchemaMetadataInfo(schemaMetadata); minTimes=0; maxTimes=1;

            mockSchemaRegistryClient.addSchemaVersion(withInstanceOf(SchemaMetadata.class), withInstanceOf(SchemaVersion.class));
            result = schemaIdVersion; minTimes=0; maxTimes=1;

            mockSchemaRegistryClient.getSchemaVersionInfo(withInstanceOf(SchemaVersionKey.class));
            result = schemaVersionInfo; minTimes=0; maxTimes=1;
        }
    };

    AvroSnapshotSerializer serializer = new AvroSnapshotSerializer();
    serializer.init(Collections.singletonMap(SERDES_PROTOCOL_VERSION, serdesProtocolVersion));

    AvroSnapshotDeserializer deserializer = new AvroSnapshotDeserializer();
    deserializer.init(Collections.emptyMap());

    byte[] serializedData = serializer.serialize(input, schemaMetadata);
    Object deserializedObj = deserializer.deserialize(new ByteArrayInputStream(serializedData), null);

    Assert.assertTrue(SpecificData.get().compare(input, deserializedObj, input.getSchema()) == 0);
}
 

public Avro17Adapter() throws Exception {
  super(
      GenericData.EnumSymbol.class.getConstructor(Schema.class, String.class),
      GenericData.Fixed.class.getConstructor(Schema.class, byte[].class)
  );
  _encoderFactory = Class.forName("org.apache.avro.io.EncoderFactory").getDeclaredMethod("get").invoke(null);
  _encoderFactoryBinaryEncoderMethod = _encoderFactory.getClass().getMethod("binaryEncoder", OutputStream.class, BinaryEncoder.class);

  Class<?> schemaParserClass = Class.forName("org.apache.avro.Schema$Parser");
  _schemaParserCtr = schemaParserClass.getConstructor();
  _setValidateMethod = schemaParserClass.getDeclaredMethod("setValidate", boolean.class);
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (runtimeVersion.laterThan(AvroVersion.AVRO_1_7)) {
    _setValidateDefaultsMethod = schemaParserClass.getDeclaredMethod("setValidateDefaults", boolean.class);
  } else {
    _setValidateDefaultsMethod = null;
  }
  _parseMethod = schemaParserClass.getDeclaredMethod("parse", String.class);
  _addTypesMethod = schemaParserClass.getDeclaredMethod("addTypes", Map.class);
  _getTypesMethod = schemaParserClass.getDeclaredMethod("getTypes");

  Class<?> schemaNormalizationClass = Class.forName("org.apache.avro.SchemaNormalization");
  _toParsingFormMethod = schemaNormalizationClass.getMethod("toParsingForm", Schema.class);

  _newInstanceMethod = SpecificData.class.getMethod("newInstance", Class.class, Schema.class);
  _getDefaultValueMethod = SpecificData.class.getMethod("getDefaultValue", Schema.Field.class);

  tryInitializeCompilerFields();
}
 

/**
 * Converts an Avro class into a nested row structure with deterministic field order and data
 * types that are compatible with Flink's Table & SQL API.
 *
 * @param avroClass Avro specific record that contains schema information
 * @return type information matching the schema
 */
@SuppressWarnings("unchecked")
public static <T extends SpecificRecord> TypeInformation<Row> convertToTypeInfo(Class<T> avroClass) {
	Preconditions.checkNotNull(avroClass, "Avro specific record class must not be null.");
	// determine schema to retrieve deterministic field order
	final Schema schema = SpecificData.get().getSchema(avroClass);
	return (TypeInformation<Row>) convertToTypeInfo(schema);
}
 

/**
 * Extracts an Avro {@link Schema} from a {@link SpecificRecord}. We do this either via {@link
 * SpecificData} or by instantiating a record and extracting the schema from the instance.
 */
static <T> Schema extractAvroSpecificSchema(
		Class<T> type,
		SpecificData specificData) {
	Optional<Schema> newSchemaOptional = tryExtractAvroSchemaViaInstance(type);
	return newSchemaOptional.orElseGet(() -> specificData.getSchema(type));
}
 

/**
 * Create a new Event Reader
 * @param in
 * @throws IOException
 */
@SuppressWarnings("deprecation")
public EventReader(DataInputStream in) throws IOException {
  this.in = in;
  this.version = in.readLine();
  
  if (!EventWriter.VERSION.equals(version)) {
    throw new IOException("Incompatible event log version: "+version);
  }

  Schema myschema = new SpecificData(Event.class.getClassLoader()).getSchema(Event.class);
  this.schema = Schema.parse(in.readLine());
  this.reader = new SpecificDatumReader(schema, myschema);
  this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
}
 

@Test
public void testWriteParquetAvroSpecific() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final List<Address> data = Arrays.asList(
			new Address(1, "a", "b", "c", "12345"),
			new Address(2, "p", "q", "r", "12345"),
			new Address(3, "x", "y", "z", "12345")
	);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Address> stream = env.addSource(
			new FiniteTestSource<>(data), TypeInformation.of(Address.class));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forSpecificRecord(Address.class))
			.build());

	env.execute();

	validateResults(folder, SpecificData.get(), data);
}
 

/**
 * Creates an Avro serialization schema for the given specific record class.
 *
 * @param recordClazz Avro record class used to serialize Flink's row to Avro's record
 */
public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) {
	Preconditions.checkNotNull(recordClazz, "Avro record class must not be null.");
	this.recordClazz = recordClazz;
	this.schema = SpecificData.get().getSchema(recordClazz);
	this.schemaString = schema.toString();
	this.datumWriter = new SpecificDatumWriter<>(schema);
	this.arrayOutputStream = new ByteArrayOutputStream();
	this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
 

@Test
public void testWriteParquetAvroGeneric() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final Schema schema = Address.getClassSchema();

	final Collection<GenericRecord> data = new GenericTestDataCollection();

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<GenericRecord> stream = env.addSource(
			new FiniteTestSource<>(data), new GenericRecordAvroTypeInfo(schema));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forGenericRecord(schema))
					.build());

	env.execute();

	List<Address> expected = Arrays.asList(
			new Address(1, "a", "b", "c", "12345"),
			new Address(2, "x", "y", "z", "98765"));

	validateResults(folder, SpecificData.get(), expected);
}
 

@Test
public void testWriteParquetAvroSpecific() throws Exception {

	final File folder = TEMPORARY_FOLDER.newFolder();

	final List<Address> data = Arrays.asList(
			new Address(1, "a", "b", "c", "12345"),
			new Address(2, "p", "q", "r", "12345"),
			new Address(3, "x", "y", "z", "12345")
	);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Address> stream = env.addSource(
			new FiniteTestSource<>(data), TypeInformation.of(Address.class));

	stream.addSink(
			StreamingFileSink.forBulkFormat(
					Path.fromLocalFile(folder),
					ParquetAvroWriters.forSpecificRecord(Address.class))
			.build());

	env.execute();

	validateResults(folder, SpecificData.get(), data);
}
 

@SuppressWarnings("unchecked")
private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException {
	recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject();
	schemaString = (String) inputStream.readObject();
	if (recordClazz != null) {
		schema = SpecificData.get().getSchema(recordClazz);
	} else {
		schema = new Schema.Parser().parse(schemaString);
	}
	datumWriter = new SpecificDatumWriter<>(schema);
	arrayOutputStream = new ByteArrayOutputStream();
	encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 

/**
 * Creates an Avro serialization schema for the given specific record class.
 *
 * @param recordClazz Avro record class used to serialize Flink's row to Avro's record
 */
public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) {
	Preconditions.checkNotNull(recordClazz, "Avro record class must not be null.");
	this.recordClazz = recordClazz;
	this.schema = SpecificData.get().getSchema(recordClazz);
	this.schemaString = schema.toString();
	this.datumWriter = new SpecificDatumWriter<>(schema);
	this.arrayOutputStream = new ByteArrayOutputStream();
	this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null);
}
 

/**
 * Get the DatumReader for the given type.
 *
 * @param <E> The entity type
 * @param type The Java class of the entity type
 * @param writerSchema The {@link Schema} for entities
 * @return The DatumReader for the given type
 */
@SuppressWarnings("unchecked")
public static <E> DatumReader<E> getDatumReaderForType(Class<E> type, Schema writerSchema) {
  Schema readerSchema = getReaderSchema(type, writerSchema);
  GenericData dataModel = getDataModelForType(type);
  if (dataModel instanceof ReflectData) {
    return new ReflectDatumReader<E>(writerSchema, readerSchema, (ReflectData)dataModel);
  } else if (dataModel instanceof SpecificData) {
    return new SpecificDatumReader<E>(writerSchema, readerSchema, (SpecificData)dataModel);
  } else {
    return new GenericDatumReader<E>(writerSchema, readerSchema, dataModel);
  }
}
 

@Override
public Object getSpecificDefaultValue(Schema.Field field) {
  if (getSpecificDefaultValueMethod != null) {
    return SpecificData.get().getDefaultValue(field);
  } else {
    //old avro 1.7 - punt to backported code
    return Avro17DefaultValuesCache.getDefaultValue(field, true);
  }
}
 

private void roundtrip(Object thingie) throws Exception {
  Schema schema = SpecificData.get().getSchema(thingie.getClass());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  BinaryEncoder binaryEncoder = AvroCompatibilityHelper.newBinaryEncoder(os, false, null);
  SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(schema);
  writer.write(thingie, binaryEncoder);
  binaryEncoder.flush();
  byte[] serialized = os.toByteArray();
  ByteArrayInputStream is = new ByteArrayInputStream(serialized);
  BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is, false, null);
  SpecificDatumReader<Object> reader = new SpecificDatumReader<>(schema);
  Object deserialize = reader.read(null, binaryDecoder);
  Assert.assertEquals(deserialize, thingie);
}
 

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) {
	SpecificData specificData = new SpecificData(cl);
	Schema newSchema = specificData.getSchema(type);

	return new AvroFactory<>(
		specificData,
		newSchema,
		new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData),
		new SpecificDatumWriter<>(newSchema, specificData)
	);
}
 

@Override public void readExternal(java.io.ObjectInput in)
  throws java.io.IOException {
  READER$.read(this, SpecificData.getDecoder(in));
}
 

@Override public void readExternal(java.io.ObjectInput in)
  throws java.io.IOException {
  READER$.read(this, SpecificData.getDecoder(in));
}
 

@Override public void writeExternal(java.io.ObjectOutput out)
  throws java.io.IOException {
  WRITER$.write(this, SpecificData.getEncoder(out));
}
 

@Override public void readExternal(java.io.ObjectInput in)
  throws java.io.IOException {
  READER$.read(this, SpecificData.getDecoder(in));
}