Java Code Examples for org.apache.avro.Schema#parse()

@SuppressWarnings("rawtypes")
@Override
public OutputFormat getOutputFormat() throws IOException {
    AvroStorageLog.funcCall("getOutputFormat");

    Properties property = getUDFProperties();
    String allSchemaStr = property.getProperty(AVRO_OUTPUT_SCHEMA_PROPERTY);
    Map<String, String> map = (allSchemaStr != null)  ? parseSchemaMap(allSchemaStr) : null;

    String key = getSchemaKey();
    Schema schema = (map == null || !map.containsKey(key))  ? outputAvroSchema  : Schema.parse(map.get(key));

    if (schema == null)
        throw new IOException("Output schema is null!");
    AvroStorageLog.details("Output schema=" + schema);

    return new PigAvroOutputFormat(schema);
}
 

@Test(groups = {"deserializationTest"}, dataProvider = "SlowFastDeserializer")
public void javaStringPropertyTest(Boolean whetherUseFastDeserializer) throws IOException {
  Schema javaStringSchema = Schema.parse("{\n" + "  \"type\":\"string\",\n" + "  \"avro.java.string\":\"String\"} ");
  Schema schema = createRecord(createField("testString", javaStringSchema));

  GenericRecord record = new GenericData.Record(schema);
  record.put("testString", "aaa");

  Decoder decoder = writeWithFastAvro(record, schema, false);

  GenericRecord afterDecoding;
  if (whetherUseFastDeserializer) {
    afterDecoding = readWithFastAvro(schema, schema, decoder, false);
  } else {
    afterDecoding = readWithSlowAvro(schema, schema, decoder, false);
  }

  if (Utils.isAvro14()){
    Assert.assertTrue(afterDecoding.get(0) instanceof Utf8, "Utf8 is expected, but got: " + afterDecoding.get(0).getClass());
  }  else {
    Assert.assertTrue(afterDecoding.get(0) instanceof String, "String is expected, but got: " + afterDecoding.get(0).getClass());
  }
}
 

@Test
public void testEncodeAvroIndexedRecordAsJson() throws Exception {
  String expectedValue = "name123";
  Schema schema = Schema.parse(SCHEMA_STRING);
  GenericRecord record = new GenericData.Record(schema);
  record.put("first", expectedValue);
  String json = new String(encodeAvroIndexedRecordAsJson(schema, record));
  Assert.assertTrue(json.contains(expectedValue));
}
 

/**
 * This method is called to return the schema of an avro schema file. This
 * method is different than {@link #getSchema}, which returns the schema
 * from a data file.
 *
 * @param path  path of a file or first level directory
 * @param fs  file system
 * @return avro schema
 * @throws IOException
 */
protected Schema getSchemaFromFile(Path path, FileSystem fs) throws IOException {
    /* get path of the last file */
    Path lastFile = AvroStorageUtils.getLast(path, fs);
    if (lastFile == null) {
        return null;
    }

    /* read in file and obtain schema */
    GenericDatumReader<Object> avroReader = new GenericDatumReader<Object>();
    InputStream hdfsInputStream = fs.open(lastFile);
    Schema ret = Schema.parse(hdfsInputStream);
    hdfsInputStream.close();

    return ret;
}
 

private void validateProperties() {
	String keySchemaString = properties.get(CONF_OUTPUT_KEY_SCHEMA);
	if (keySchemaString == null) {
		throw new IllegalStateException("No key schema provided, set '" + CONF_OUTPUT_KEY_SCHEMA + "' property");
	}
	Schema.parse(keySchemaString); //verifying that schema valid

	String valueSchemaString = properties.get(CONF_OUTPUT_VALUE_SCHEMA);
	if (valueSchemaString == null) {
		throw new IllegalStateException("No value schema provided, set '" + CONF_OUTPUT_VALUE_SCHEMA + "' property");
	}
	Schema.parse(valueSchemaString); //verifying that schema valid
}
 

@Test
public void testAvro14DoesntValidateNamespace() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (runtimeVersion != AvroVersion.AVRO_1_4) {
    throw new SkipException("only supported under " + AvroVersion.AVRO_1_4 + ". runtime version detected as " + runtimeVersion);
  }
  String withAvsc = TestUtil.load("HasNamespace.avsc");
  Schema with = Schema.parse(withAvsc);
  String withoutAvsc = TestUtil.load("HasNoNamespace.avsc");
  Schema without = Schema.parse(withoutAvsc);

  GenericData.Record record = new GenericData.Record(without);
  record.put("f", AvroCompatibilityHelper.newEnumSymbol(without.getField("f").schema(), "B"));

  ByteArrayOutputStream os = new ByteArrayOutputStream();
  GenericDatumWriter writer = new GenericDatumWriter(without);
  BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(os);
  //noinspection unchecked
  writer.write(record, encoder);
  encoder.flush();
  byte[] bytes = os.toByteArray();

  GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<>(without, with);
  BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
  GenericData.Record read = reader.read(null, decoder);

  String value = String.valueOf(read.get("f"));
  Assert.assertEquals(value, "B");
}
 

private void validateProperties() {
	String keySchemaString = properties.get(CONF_OUTPUT_KEY_SCHEMA);
	if (keySchemaString == null) {
		throw new IllegalStateException("No key schema provided, set '" + CONF_OUTPUT_KEY_SCHEMA + "' property");
	}
	Schema.parse(keySchemaString); //verifying that schema valid

	String valueSchemaString = properties.get(CONF_OUTPUT_VALUE_SCHEMA);
	if (valueSchemaString == null) {
		throw new IllegalStateException("No value schema provided, set '" + CONF_OUTPUT_VALUE_SCHEMA + "' property");
	}
	Schema.parse(valueSchemaString); //verifying that schema valid
}
 

@Test (groups = "serializationTest")
public void testIsSupportedForFastSpecificDatumWriter() {
  Schema testSchema = Schema.parse("{\"type\": \"record\", \"name\": \"test_record\", \"fields\":[]}");
  FastSpecificDatumWriter fastWriter = FastDatumReaderWriterUtil.getFastSpecificDatumWriter(testSchema);
  Assert.assertNotNull(fastWriter);
  FastSpecificDatumWriter newFastWriter = FastDatumReaderWriterUtil.getFastSpecificDatumWriter(testSchema);
  Assert.assertSame(fastWriter, newFastWriter);
}
 

@Override
public String convertSchema(String inputSchema, WorkUnitState workUnit)
    throws SchemaConversionException {
  Schema schema = Schema.parse(inputSchema);
  if (null == schemaId) {
    try {
      schemaId = getSchemaId(workUnit.getProperties(), schema);
    } catch (SchemaRegistryException e) {
      throw new SchemaConversionException(e);
    }
  }
  return schema.toString();
}
 

@Test(expected = DatasetException.class)
public void testPutWithNullKey() throws Exception {
  Dao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
      schemaString);
  @SuppressWarnings("deprecation")
  GenericRecord entity = new GenericData.Record(Schema.parse(schemaString));
  entity.put("keyPart1", "part1");
  entity.put("keyPart2", null);
  entity.put("field1", "field1");
  entity.put("field2", "field2");
  dao.put(entity);
}
 

private Schema computePayloadSchema(String streamName, SamzaSqlRelMessage relMessage) {
  SamzaSqlRelRecord relRecord = relMessage.getSamzaSqlRelRecord();
  List<Schema.Field> keyFields = new ArrayList<>();
  List<String> fieldNames = relRecord.getFieldNames();
  List<Object> values = relRecord.getFieldValues();

  for (int index = 0; index < fieldNames.size(); index++) {
    if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
      continue;
    }

    Object value = values.get(index);
    Schema avroType;
    if (value instanceof GenericData.Record) {
      avroType = ((GenericData.Record) value).getSchema();
    } else {
      avroType = ReflectData.get().getSchema(value.getClass());
    }
    keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
  }

  Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
  ks.setFields(keyFields);
  String schemaStr = ks.toString();
  Schema schema;
  // See whether we have a schema object corresponding to the schemaValue and reuse it.
  // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
  if (schemas.containsKey(schemaStr)) {
    schema = schemas.get(schemaStr);
  } else {
    schema = Schema.parse(schemaStr);
    schemas.put(schemaStr, schema);
  }

  return schema;
}
 

@Test(expectedExceptions = FastDeserializerGeneratorException.class, groups = {"deserializationTest"})
public void shouldNotReadStrippedEnum() throws IOException {
  // given
  Schema oldRecordSchema =
      Schema.parse(this.getClass().getResourceAsStream("/schema/fastserdetestoldextendedenum.avsc"));
  GenericData.Fixed testFixed = newFixed(oldRecordSchema.getField("testFixed").schema(), new byte[]{0x01});
  GenericData.Record subRecord =
      new GenericData.Record(oldRecordSchema.getField("subRecordUnion").schema().getTypes().get(1));
  GenericData.Record oldRecord = new GenericData.Record(oldRecordSchema);
  oldRecord.put("testInt", 1);
  oldRecord.put("testLong", 1l);
  oldRecord.put("testDouble", 1.0);
  oldRecord.put("testFloat", 1.0f);
  oldRecord.put("testBoolean", true);
  oldRecord.put("testBytes", ByteBuffer.wrap(new byte[]{0x01, 0x02}));
  oldRecord.put("testString", "aaa");
  oldRecord.put("testFixed", testFixed);
  oldRecord.put("testFixedUnion", testFixed);
  oldRecord.put("testFixedArray", Arrays.asList(testFixed));
  oldRecord.put("testFixedUnionArray", Arrays.asList(testFixed));
  oldRecord.put("testEnum",
      AvroCompatibilityHelper.newEnumSymbol(SCHEMA_FOR_TEST_ENUM, "F"));//new GenericData.EnumSymbol("F"));
  oldRecord.put("testEnumArray", Arrays.asList(
      AvroCompatibilityHelper.newEnumSymbol(SCHEMA_FOR_TEST_ENUM, "F"))); //new GenericData.EnumSymbol("F")));
  oldRecord.put("testEnumUnionArray", Collections.emptyList());

  oldRecord.put("subRecordUnion", subRecord);
  oldRecord.put("subRecord", subRecord);

  oldRecord.put("recordsArray", Collections.emptyList());
  oldRecord.put("recordsArrayMap", Collections.emptyList());
  oldRecord.put("recordsMap", Collections.emptyMap());
  oldRecord.put("recordsMapArray", Collections.emptyMap());

  // when
  TestRecord record = decodeRecordFast(TestRecord.SCHEMA$, oldRecordSchema, genericDataAsDecoder(oldRecord));
}
 

@Test
public void testMapToEntity() throws Exception {
  AvroKeySchema keySchema = schemaParser.parseKeySchema(schemaString);
  AvroEntitySchema entitySchema = schemaParser.parseEntitySchema(schemaString);

  AvroKeySerDe keySerDe = new AvroKeySerDe(
      keySchema.getAvroSchema(), keySchema.getPartitionStrategy());
  AvroEntitySerDe<GenericRecord> entitySerDe = new AvroEntitySerDe<GenericRecord>(
      new AvroEntityComposer<GenericRecord>(entitySchema, false),
      entitySchema, entitySchema, false);
  EntityMapper<GenericRecord> entityMapper = new BaseEntityMapper<GenericRecord>(
      keySchema, entitySchema, keySerDe, entitySerDe);

  byte[] row = new byte[] { (byte) 0x80, (byte) 0, (byte) 0, (byte) 0, // hash
      (byte) 0x80, (byte) 0, (byte) 0, (byte) 2,   // keyPart2
      (byte) 0x80, (byte) 0, (byte) 0, (byte) 1 }; // keyPart1

  byte[] intFamily = stringToBytes("int");
  byte[] mapFamily = stringToBytes("map");
  byte[] recordFamily = stringToBytes("record");
  byte[] intQual1 = stringToBytes("1");
  byte[] intQual2 = stringToBytes("2");
  byte[] mapQual1 = stringToBytes("1");
  byte[] mapQual2 = stringToBytes("2");
  byte[] mapQual3 = stringToBytes("3");
  byte[] recordQual1 = stringToBytes("sub_field1");
  byte[] recordQual2 = stringToBytes("sub_field2");

  byte[] intValue1 = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 1 };
  byte[] intValue2 = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 2 };
  byte[] recordIntValue1 = new byte[] { (byte) 1 };
  byte[] recordIntValue2 = new byte[] { (byte) 2 };
  @SuppressWarnings("deprecation")
  Schema strSchema = Schema.parse("{ \"type\": \"string\" }");
  DatumWriter<Utf8> datumWriter = new GenericDatumWriter<Utf8>(strSchema);
  byte[] stringValue1 = AvroUtils.writeAvroEntity(new Utf8("string_value1"),
      datumWriter);
  byte[] stringValue2 = AvroUtils.writeAvroEntity(new Utf8("string_value2"),
      datumWriter);
  byte[] stringValue3 = AvroUtils.writeAvroEntity(new Utf8("string_value3"),
      datumWriter);

  KeyValue[] keyValues = new KeyValue[] {
      new KeyValue(row, intFamily, intQual1, intValue1),
      new KeyValue(row, intFamily, intQual2, intValue2),
      new KeyValue(row, mapFamily, mapQual1, stringValue1),
      new KeyValue(row, mapFamily, mapQual2, stringValue2),
      new KeyValue(row, mapFamily, mapQual3, stringValue3),
      new KeyValue(row, recordFamily, recordQual1, recordIntValue1),
      new KeyValue(row, recordFamily, recordQual2, recordIntValue2) };
  Result result = new Result(keyValues);

  GenericRecord entity = entityMapper
      .mapToEntity(result);

  assertEquals(1, entity.get("keyPart1"));
  assertEquals(2, entity.get("keyPart2"));
  assertEquals(1, entity.get("field1"));
  assertEquals(2, entity.get("field2"));

  @SuppressWarnings("unchecked")
  Map<CharSequence, Utf8> field3 = (Map<CharSequence, Utf8>) entity
      .get("field3");
  assertEquals("string_value1", field3.get(new Utf8("1")).toString());
  assertEquals("string_value2", field3.get(new Utf8("2")).toString());
  assertEquals("string_value3", field3.get(new Utf8("3")).toString());

  GenericRecord field4 = (GenericRecord) entity.get("field4");
  assertEquals(-1, ((Integer) field4.get("sub_field1")).intValue());
  assertEquals(1, ((Integer) field4.get("sub_field2")).intValue());
}
 

@Test
public void testFixedField() throws Exception {
  Schema fixedTypeSchema = Schema.parse("{\"name\": \"UUID\", \"type\":\"fixed\", \"size\":16}");
  GenericData.Fixed fixed = AvroCompatibilityHelper.newFixedField(fixedTypeSchema);
  Assert.assertNotNull(fixed);
}
 

@Test
public void demonstrateAvro14DoesntValidateFieldNames() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 

public static void assertValidSchema(String schemaJson) throws Exception {
  Schema.parse(schemaJson);
}
 

@Test
public void demonstrateAvro14DoesntValidateFixedNames() throws Exception {
  String avsc = TestUtil.load("FixedWithInvalidName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 

@Test
public void demonstrateAvro14DoesntValidateEnumNames() throws Exception {
  String avsc = TestUtil.load("EnumWithInvalidName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 

@Test(groups = {"deserializationTest"})
public void testFastGenericDeserializerPrimitFloatList() throws Exception {
  String schemaString = "{\"type\":\"record\",\"name\":\"KeyRecord\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"doc\":\"name field\"}, {\"name\":\"inventory\", \"type\" : {  \"type\" : \"array\", \"items\" : \"float\" }}] }";

  Schema oldRecordSchema = Schema.parse(schemaString);
  FastSerdeCache cache = FastSerdeCache.getDefaultInstance();
  FastDeserializer<GenericRecord> deserializer =
      (FastDeserializer<GenericRecord>) cache.buildFastGenericDeserializer(oldRecordSchema, oldRecordSchema);

  GenericData.Record record = new GenericData.Record(oldRecordSchema);
  ArrayList<Float> arrayList = new ArrayList();
  arrayList.add((float)10);
  arrayList.add((float)20);

  record.put("name", "test");
  record.put("inventory", arrayList);
  byte[] serializedBytes = serialize(record, oldRecordSchema);
  GenericRecord deserRecord = deserializer.deserialize(getDecoder(serializedBytes));

  // Generate a different record
  // new record array length larger than reuse record.
  GenericData.Record record1 = new GenericData.Record(oldRecordSchema);
  record1.put("name", "test1");
  arrayList.add((float)30);
  record1.put("inventory", arrayList);
  serializedBytes = serialize(record1, oldRecordSchema);
  // generate a record to reuse bytebuffer
  GenericRecord genericRecord = deserializer.deserialize(deserRecord, getDecoder(serializedBytes));
  List<Float> list = (List<Float>)genericRecord.get(1);
  Assert.assertEquals(list.size(), 3);

  // new record array length same as before for reusing byte buffers
  arrayList.clear();
  arrayList.add((float)10);
  arrayList.add((float)20);
  arrayList.add((float)30);
  record1.put("inventory", arrayList);
  serializedBytes = serialize(record1, oldRecordSchema);
  genericRecord = deserializer.deserialize(deserRecord, getDecoder(serializedBytes));
  list = (List<Float>)genericRecord.get(1);
  Assert.assertEquals(list.size(), 3);

}
 

public void generateAndValidateSchema(Record record, String fieldFragment) throws OnRecordErrorException {
  String schema = generator.generateSchema(record);
  Assert.assertNotNull(schema);
  Schema expectedSchema = Schema.parse("{\"type\":\"record\",\"name\":\"test_schema\",\"fields\":[" + fieldFragment + "]}");
  Assert.assertEquals(expectedSchema, Schema.parse(schema));
}