Java Code Examples for org.apache.avro.Schema#parse()

The following examples show how to use org.apache.avro.Schema#parse() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Cubert   File: AvroStorage.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
public OutputFormat getOutputFormat() throws IOException {
    AvroStorageLog.funcCall("getOutputFormat");

    Properties property = getUDFProperties();
    String allSchemaStr = property.getProperty(AVRO_OUTPUT_SCHEMA_PROPERTY);
    Map<String, String> map = (allSchemaStr != null)  ? parseSchemaMap(allSchemaStr) : null;

    String key = getSchemaKey();
    Schema schema = (map == null || !map.containsKey(key))  ? outputAvroSchema  : Schema.parse(map.get(key));

    if (schema == null)
        throw new IOException("Output schema is null!");
    AvroStorageLog.details("Output schema=" + schema);

    return new PigAvroOutputFormat(schema);
}
 
Example 2
@Test(groups = {"deserializationTest"}, dataProvider = "SlowFastDeserializer")
public void javaStringPropertyTest(Boolean whetherUseFastDeserializer) throws IOException {
  Schema javaStringSchema = Schema.parse("{\n" + "  \"type\":\"string\",\n" + "  \"avro.java.string\":\"String\"} ");
  Schema schema = createRecord(createField("testString", javaStringSchema));

  GenericRecord record = new GenericData.Record(schema);
  record.put("testString", "aaa");

  Decoder decoder = writeWithFastAvro(record, schema, false);

  GenericRecord afterDecoding;
  if (whetherUseFastDeserializer) {
    afterDecoding = readWithFastAvro(schema, schema, decoder, false);
  } else {
    afterDecoding = readWithSlowAvro(schema, schema, decoder, false);
  }

  if (Utils.isAvro14()){
    Assert.assertTrue(afterDecoding.get(0) instanceof Utf8, "Utf8 is expected, but got: " + afterDecoding.get(0).getClass());
  }  else {
    Assert.assertTrue(afterDecoding.get(0) instanceof String, "String is expected, but got: " + afterDecoding.get(0).getClass());
  }
}
 
Example 3
@Test
public void testEncodeAvroIndexedRecordAsJson() throws Exception {
  String expectedValue = "name123";
  Schema schema = Schema.parse(SCHEMA_STRING);
  GenericRecord record = new GenericData.Record(schema);
  record.put("first", expectedValue);
  String json = new String(encodeAvroIndexedRecordAsJson(schema, record));
  Assert.assertTrue(json.contains(expectedValue));
}
 
Example 4
Source Project: spork   File: AvroStorage.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method is called to return the schema of an avro schema file. This
 * method is different than {@link #getSchema}, which returns the schema
 * from a data file.
 *
 * @param path  path of a file or first level directory
 * @param fs  file system
 * @return avro schema
 * @throws IOException
 */
protected Schema getSchemaFromFile(Path path, FileSystem fs) throws IOException {
    /* get path of the last file */
    Path lastFile = AvroStorageUtils.getLast(path, fs);
    if (lastFile == null) {
        return null;
    }

    /* read in file and obtain schema */
    GenericDatumReader<Object> avroReader = new GenericDatumReader<Object>();
    InputStream hdfsInputStream = fs.open(lastFile);
    Schema ret = Schema.parse(hdfsInputStream);
    hdfsInputStream.close();

    return ret;
}
 
Example 5
private void validateProperties() {
	String keySchemaString = properties.get(CONF_OUTPUT_KEY_SCHEMA);
	if (keySchemaString == null) {
		throw new IllegalStateException("No key schema provided, set '" + CONF_OUTPUT_KEY_SCHEMA + "' property");
	}
	Schema.parse(keySchemaString); //verifying that schema valid

	String valueSchemaString = properties.get(CONF_OUTPUT_VALUE_SCHEMA);
	if (valueSchemaString == null) {
		throw new IllegalStateException("No value schema provided, set '" + CONF_OUTPUT_VALUE_SCHEMA + "' property");
	}
	Schema.parse(valueSchemaString); //verifying that schema valid
}
 
Example 6
@Test
public void testAvro14DoesntValidateNamespace() throws Exception {
  AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
  if (runtimeVersion != AvroVersion.AVRO_1_4) {
    throw new SkipException("only supported under " + AvroVersion.AVRO_1_4 + ". runtime version detected as " + runtimeVersion);
  }
  String withAvsc = TestUtil.load("HasNamespace.avsc");
  Schema with = Schema.parse(withAvsc);
  String withoutAvsc = TestUtil.load("HasNoNamespace.avsc");
  Schema without = Schema.parse(withoutAvsc);

  GenericData.Record record = new GenericData.Record(without);
  record.put("f", AvroCompatibilityHelper.newEnumSymbol(without.getField("f").schema(), "B"));

  ByteArrayOutputStream os = new ByteArrayOutputStream();
  GenericDatumWriter writer = new GenericDatumWriter(without);
  BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(os);
  //noinspection unchecked
  writer.write(record, encoder);
  encoder.flush();
  byte[] bytes = os.toByteArray();

  GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<>(without, with);
  BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
  GenericData.Record read = reader.read(null, decoder);

  String value = String.valueOf(read.get("f"));
  Assert.assertEquals(value, "B");
}
 
Example 7
Source Project: flink   File: AvroKeyValueSinkWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void validateProperties() {
	String keySchemaString = properties.get(CONF_OUTPUT_KEY_SCHEMA);
	if (keySchemaString == null) {
		throw new IllegalStateException("No key schema provided, set '" + CONF_OUTPUT_KEY_SCHEMA + "' property");
	}
	Schema.parse(keySchemaString); //verifying that schema valid

	String valueSchemaString = properties.get(CONF_OUTPUT_VALUE_SCHEMA);
	if (valueSchemaString == null) {
		throw new IllegalStateException("No value schema provided, set '" + CONF_OUTPUT_VALUE_SCHEMA + "' property");
	}
	Schema.parse(valueSchemaString); //verifying that schema valid
}
 
Example 8
@Test (groups = "serializationTest")
public void testIsSupportedForFastSpecificDatumWriter() {
  Schema testSchema = Schema.parse("{\"type\": \"record\", \"name\": \"test_record\", \"fields\":[]}");
  FastSpecificDatumWriter fastWriter = FastDatumReaderWriterUtil.getFastSpecificDatumWriter(testSchema);
  Assert.assertNotNull(fastWriter);
  FastSpecificDatumWriter newFastWriter = FastDatumReaderWriterUtil.getFastSpecificDatumWriter(testSchema);
  Assert.assertSame(fastWriter, newFastWriter);
}
 
Example 9
@Override
public String convertSchema(String inputSchema, WorkUnitState workUnit)
    throws SchemaConversionException {
  Schema schema = Schema.parse(inputSchema);
  if (null == schemaId) {
    try {
      schemaId = getSchemaId(workUnit.getProperties(), schema);
    } catch (SchemaRegistryException e) {
      throw new SchemaConversionException(e);
    }
  }
  return schema.toString();
}
 
Example 10
Source Project: kite   File: AvroDaoTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = DatasetException.class)
public void testPutWithNullKey() throws Exception {
  Dao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
      schemaString);
  @SuppressWarnings("deprecation")
  GenericRecord entity = new GenericData.Record(Schema.parse(schemaString));
  entity.put("keyPart1", "part1");
  entity.put("keyPart2", null);
  entity.put("field1", "field1");
  entity.put("field2", "field2");
  dao.put(entity);
}
 
Example 11
Source Project: samza   File: AvroSchemaGenRelConverter.java    License: Apache License 2.0 5 votes vote down vote up
private Schema computePayloadSchema(String streamName, SamzaSqlRelMessage relMessage) {
  SamzaSqlRelRecord relRecord = relMessage.getSamzaSqlRelRecord();
  List<Schema.Field> keyFields = new ArrayList<>();
  List<String> fieldNames = relRecord.getFieldNames();
  List<Object> values = relRecord.getFieldValues();

  for (int index = 0; index < fieldNames.size(); index++) {
    if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
      continue;
    }

    Object value = values.get(index);
    Schema avroType;
    if (value instanceof GenericData.Record) {
      avroType = ((GenericData.Record) value).getSchema();
    } else {
      avroType = ReflectData.get().getSchema(value.getClass());
    }
    keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
  }

  Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
  ks.setFields(keyFields);
  String schemaStr = ks.toString();
  Schema schema;
  // See whether we have a schema object corresponding to the schemaValue and reuse it.
  // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
  if (schemas.containsKey(schemaStr)) {
    schema = schemas.get(schemaStr);
  } else {
    schema = Schema.parse(schemaStr);
    schemas.put(schemaStr, schema);
  }

  return schema;
}
 
Example 12
@Test(expectedExceptions = FastDeserializerGeneratorException.class, groups = {"deserializationTest"})
public void shouldNotReadStrippedEnum() throws IOException {
  // given
  Schema oldRecordSchema =
      Schema.parse(this.getClass().getResourceAsStream("/schema/fastserdetestoldextendedenum.avsc"));
  GenericData.Fixed testFixed = newFixed(oldRecordSchema.getField("testFixed").schema(), new byte[]{0x01});
  GenericData.Record subRecord =
      new GenericData.Record(oldRecordSchema.getField("subRecordUnion").schema().getTypes().get(1));
  GenericData.Record oldRecord = new GenericData.Record(oldRecordSchema);
  oldRecord.put("testInt", 1);
  oldRecord.put("testLong", 1l);
  oldRecord.put("testDouble", 1.0);
  oldRecord.put("testFloat", 1.0f);
  oldRecord.put("testBoolean", true);
  oldRecord.put("testBytes", ByteBuffer.wrap(new byte[]{0x01, 0x02}));
  oldRecord.put("testString", "aaa");
  oldRecord.put("testFixed", testFixed);
  oldRecord.put("testFixedUnion", testFixed);
  oldRecord.put("testFixedArray", Arrays.asList(testFixed));
  oldRecord.put("testFixedUnionArray", Arrays.asList(testFixed));
  oldRecord.put("testEnum",
      AvroCompatibilityHelper.newEnumSymbol(SCHEMA_FOR_TEST_ENUM, "F"));//new GenericData.EnumSymbol("F"));
  oldRecord.put("testEnumArray", Arrays.asList(
      AvroCompatibilityHelper.newEnumSymbol(SCHEMA_FOR_TEST_ENUM, "F"))); //new GenericData.EnumSymbol("F")));
  oldRecord.put("testEnumUnionArray", Collections.emptyList());

  oldRecord.put("subRecordUnion", subRecord);
  oldRecord.put("subRecord", subRecord);

  oldRecord.put("recordsArray", Collections.emptyList());
  oldRecord.put("recordsArrayMap", Collections.emptyList());
  oldRecord.put("recordsMap", Collections.emptyMap());
  oldRecord.put("recordsMapArray", Collections.emptyMap());

  // when
  TestRecord record = decodeRecordFast(TestRecord.SCHEMA$, oldRecordSchema, genericDataAsDecoder(oldRecord));
}
 
Example 13
Source Project: kite   File: AvroEntityMapperTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapToEntity() throws Exception {
  AvroKeySchema keySchema = schemaParser.parseKeySchema(schemaString);
  AvroEntitySchema entitySchema = schemaParser.parseEntitySchema(schemaString);

  AvroKeySerDe keySerDe = new AvroKeySerDe(
      keySchema.getAvroSchema(), keySchema.getPartitionStrategy());
  AvroEntitySerDe<GenericRecord> entitySerDe = new AvroEntitySerDe<GenericRecord>(
      new AvroEntityComposer<GenericRecord>(entitySchema, false),
      entitySchema, entitySchema, false);
  EntityMapper<GenericRecord> entityMapper = new BaseEntityMapper<GenericRecord>(
      keySchema, entitySchema, keySerDe, entitySerDe);

  byte[] row = new byte[] { (byte) 0x80, (byte) 0, (byte) 0, (byte) 0, // hash
      (byte) 0x80, (byte) 0, (byte) 0, (byte) 2,   // keyPart2
      (byte) 0x80, (byte) 0, (byte) 0, (byte) 1 }; // keyPart1

  byte[] intFamily = stringToBytes("int");
  byte[] mapFamily = stringToBytes("map");
  byte[] recordFamily = stringToBytes("record");
  byte[] intQual1 = stringToBytes("1");
  byte[] intQual2 = stringToBytes("2");
  byte[] mapQual1 = stringToBytes("1");
  byte[] mapQual2 = stringToBytes("2");
  byte[] mapQual3 = stringToBytes("3");
  byte[] recordQual1 = stringToBytes("sub_field1");
  byte[] recordQual2 = stringToBytes("sub_field2");

  byte[] intValue1 = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 1 };
  byte[] intValue2 = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 2 };
  byte[] recordIntValue1 = new byte[] { (byte) 1 };
  byte[] recordIntValue2 = new byte[] { (byte) 2 };
  @SuppressWarnings("deprecation")
  Schema strSchema = Schema.parse("{ \"type\": \"string\" }");
  DatumWriter<Utf8> datumWriter = new GenericDatumWriter<Utf8>(strSchema);
  byte[] stringValue1 = AvroUtils.writeAvroEntity(new Utf8("string_value1"),
      datumWriter);
  byte[] stringValue2 = AvroUtils.writeAvroEntity(new Utf8("string_value2"),
      datumWriter);
  byte[] stringValue3 = AvroUtils.writeAvroEntity(new Utf8("string_value3"),
      datumWriter);

  KeyValue[] keyValues = new KeyValue[] {
      new KeyValue(row, intFamily, intQual1, intValue1),
      new KeyValue(row, intFamily, intQual2, intValue2),
      new KeyValue(row, mapFamily, mapQual1, stringValue1),
      new KeyValue(row, mapFamily, mapQual2, stringValue2),
      new KeyValue(row, mapFamily, mapQual3, stringValue3),
      new KeyValue(row, recordFamily, recordQual1, recordIntValue1),
      new KeyValue(row, recordFamily, recordQual2, recordIntValue2) };
  Result result = new Result(keyValues);

  GenericRecord entity = entityMapper
      .mapToEntity(result);

  assertEquals(1, entity.get("keyPart1"));
  assertEquals(2, entity.get("keyPart2"));
  assertEquals(1, entity.get("field1"));
  assertEquals(2, entity.get("field2"));

  @SuppressWarnings("unchecked")
  Map<CharSequence, Utf8> field3 = (Map<CharSequence, Utf8>) entity
      .get("field3");
  assertEquals("string_value1", field3.get(new Utf8("1")).toString());
  assertEquals("string_value2", field3.get(new Utf8("2")).toString());
  assertEquals("string_value3", field3.get(new Utf8("3")).toString());

  GenericRecord field4 = (GenericRecord) entity.get("field4");
  assertEquals(-1, ((Integer) field4.get("sub_field1")).intValue());
  assertEquals(1, ((Integer) field4.get("sub_field2")).intValue());
}
 
Example 14
@Test
public void testFixedField() throws Exception {
  Schema fixedTypeSchema = Schema.parse("{\"name\": \"UUID\", \"type\":\"fixed\", \"size\":16}");
  GenericData.Fixed fixed = AvroCompatibilityHelper.newFixedField(fixedTypeSchema);
  Assert.assertNotNull(fixed);
}
 
Example 15
@Test
public void demonstrateAvro14DoesntValidateFieldNames() throws Exception {
  String avsc = TestUtil.load("RecordWithInvalidFieldName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 
Example 16
public static void assertValidSchema(String schemaJson) throws Exception {
  Schema.parse(schemaJson);
}
 
Example 17
@Test
public void demonstrateAvro14DoesntValidateFixedNames() throws Exception {
  String avsc = TestUtil.load("FixedWithInvalidName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 
Example 18
@Test
public void demonstrateAvro14DoesntValidateEnumNames() throws Exception {
  String avsc = TestUtil.load("EnumWithInvalidName.avsc");
  Schema parsedByVanilla = Schema.parse(avsc);
  Assert.assertNotNull(parsedByVanilla);
}
 
Example 19
@Test(groups = {"deserializationTest"})
public void testFastGenericDeserializerPrimitFloatList() throws Exception {
  String schemaString = "{\"type\":\"record\",\"name\":\"KeyRecord\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"doc\":\"name field\"}, {\"name\":\"inventory\", \"type\" : {  \"type\" : \"array\", \"items\" : \"float\" }}] }";

  Schema oldRecordSchema = Schema.parse(schemaString);
  FastSerdeCache cache = FastSerdeCache.getDefaultInstance();
  FastDeserializer<GenericRecord> deserializer =
      (FastDeserializer<GenericRecord>) cache.buildFastGenericDeserializer(oldRecordSchema, oldRecordSchema);

  GenericData.Record record = new GenericData.Record(oldRecordSchema);
  ArrayList<Float> arrayList = new ArrayList();
  arrayList.add((float)10);
  arrayList.add((float)20);

  record.put("name", "test");
  record.put("inventory", arrayList);
  byte[] serializedBytes = serialize(record, oldRecordSchema);
  GenericRecord deserRecord = deserializer.deserialize(getDecoder(serializedBytes));

  // Generate a different record
  // new record array length larger than reuse record.
  GenericData.Record record1 = new GenericData.Record(oldRecordSchema);
  record1.put("name", "test1");
  arrayList.add((float)30);
  record1.put("inventory", arrayList);
  serializedBytes = serialize(record1, oldRecordSchema);
  // generate a record to reuse bytebuffer
  GenericRecord genericRecord = deserializer.deserialize(deserRecord, getDecoder(serializedBytes));
  List<Float> list = (List<Float>)genericRecord.get(1);
  Assert.assertEquals(list.size(), 3);

  // new record array length same as before for reusing byte buffers
  arrayList.clear();
  arrayList.add((float)10);
  arrayList.add((float)20);
  arrayList.add((float)30);
  record1.put("inventory", arrayList);
  serializedBytes = serialize(record1, oldRecordSchema);
  genericRecord = deserializer.deserialize(deserRecord, getDecoder(serializedBytes));
  list = (List<Float>)genericRecord.get(1);
  Assert.assertEquals(list.size(), 3);

}
 
Example 20
public void generateAndValidateSchema(Record record, String fieldFragment) throws OnRecordErrorException {
  String schema = generator.generateSchema(record);
  Assert.assertNotNull(schema);
  Schema expectedSchema = Schema.parse("{\"type\":\"record\",\"name\":\"test_schema\",\"fields\":[" + fieldFragment + "]}");
  Assert.assertEquals(expectedSchema, Schema.parse(schema));
}