Java Code Examples for org.apache.avro.Schema#getField()

The following examples show how to use org.apache.avro.Schema#getField() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
URI createData(
    URI tableUri,
    Schema schema,
    String hour,
    int id,
    String fieldName,
    Object data) throws IOException {
  GenericData.Record record = new GenericData.Record(schema);
  record.put("id", id);

  if (fieldName != null) {
    Schema.Field field = schema.getField(fieldName);
    Schema fieldSchema = field.schema();
    if (data instanceof Map) {
      GenericData.Record schemaRecord = new GenericData.Record(fieldSchema);
      ((Map<String, String>) data).forEach(schemaRecord::put);
      record.put(fieldName, schemaRecord);
    } else if (data != null) {
      record.put(fieldName, data);
    }
  }

  URI partition = URI.create(tableUri + "/hour=" + hour);
  String path = partition.getPath();
  File parentFolder = new File(path);
  parentFolder.mkdirs();
  File partitionFile = new File(parentFolder, "parquet0000");
  Path filePath = new Path(partitionFile.toURI());
  ParquetWriter<GenericData.Record> writer = AvroParquetWriter.<GenericData.Record>builder(filePath)
      .withSchema(schema)
      .withConf(new Configuration())
      .build();

  try {
    writer.write(record);
  } finally {
    writer.close();
  }
  return partition;
}
 
Example 2
Source Project: kite   File: AvroEntitySerDe.java    License: Apache License 2.0 6 votes vote down vote up
private void initKACRecordDatumMaps(String fieldName, Schema fieldSchema,
    Schema writtenFieldSchema) {
  Map<String, DatumReader<Object>> recordFieldReaderMap = new HashMap<String, DatumReader<Object>>();
  Map<String, DatumWriter<Object>> recordFieldWriterMap = new HashMap<String, DatumWriter<Object>>();
  kacRecordDatumReaders.put(fieldName, recordFieldReaderMap);
  kacRecordDatumWriters.put(fieldName, recordFieldWriterMap);
  for (Field recordField : fieldSchema.getFields()) {
    Field writtenRecordField = writtenFieldSchema
        .getField(recordField.name());
    if (writtenRecordField == null) {
      continue;
    }
    recordFieldReaderMap.put(recordField.name(),
        buildDatumReader(recordField.schema(), writtenRecordField.schema()));
    recordFieldWriterMap.put(recordField.name(),
        buildDatumWriter(recordField.schema()));
  }
}
 
Example 3
@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = sRegistry.inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 
Example 4
@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = SalesforceAvroRegistryString.get().inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 
Example 5
/**
 * given a (parent) schema, and a field name, find the schema for that field.
 * if the field is a union, returns the (only) non-null branch of the union
 * @param parent parent schema containing field
 * @param fieldName name of the field in question
 * @return schema of the field (or non-null union branch thereof)
 */
public static Schema findNonNullUnionBranch(Schema parent, String fieldName) {
  if (parent == null || fieldName == null || fieldName.isEmpty()) {
    throw new IllegalArgumentException("arguments must not be null/empty");
  }
  Schema.Field field = parent.getField(fieldName);
  if (field == null) {
    return null;
  }
  Schema fieldSchema = field.schema();
  Schema.Type fieldSchemaType = fieldSchema.getType();
  if (!Schema.Type.UNION.equals(fieldSchemaType)) {
    return fieldSchema; //field is not a union
  }
  List<Schema> branches = fieldSchema.getTypes();
  List<Schema> nonNullBranches = branches.stream().
    filter(schema -> !Schema.Type.NULL.equals(schema.getType())).collect(Collectors.toList());
  if (nonNullBranches.size() != 1) {
    throw new IllegalArgumentException(String.format("field %s has %d non-null union branches, where exactly 1 is expected in %s",
      fieldName, nonNullBranches.size(), parent));
  }
  return nonNullBranches.get(0);
}
 
Example 6
Source Project: xml-avro   File: ConverterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void nestedRecursiveRecords() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='node' type='type' minOccurs='0'/>" +
            "    </xs:sequence>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    Schema.Field field = schema.getField("node");
    Schema subSchema = field.schema();
    assertSame(schema, subSchema.getTypes().get(1));

    String xml = "<root><node></node></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    GenericData.Record child = (GenericData.Record) record.get("node");
    assertEquals(record.getSchema(), child.getSchema());

    assertNull(child.get("node"));
}
 
Example 7
Source Project: beam   File: BigQueryAvroUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static Stream<TableFieldSchema> mapTableFieldSchema(
    TableFieldSchema fieldSchema, Schema avroSchema) {
  Field avroFieldSchema = avroSchema.getField(fieldSchema.getName());
  if (avroFieldSchema == null) {
    return Stream.empty();
  } else if (avroFieldSchema.schema().getType() != Type.RECORD) {
    return Stream.of(fieldSchema);
  }

  List<TableFieldSchema> subSchemas =
      fieldSchema.getFields().stream()
          .flatMap(subSchema -> mapTableFieldSchema(subSchema, avroFieldSchema.schema()))
          .collect(Collectors.toList());

  TableFieldSchema output =
      new TableFieldSchema()
          .setCategories(fieldSchema.getCategories())
          .setDescription(fieldSchema.getDescription())
          .setFields(subSchemas)
          .setMode(fieldSchema.getMode())
          .setName(fieldSchema.getName())
          .setType(fieldSchema.getType());

  return Stream.of(output);
}
 
Example 8
Source Project: pulsar   File: JSONSchemaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllowNullSchema() throws JSONException {
    JSONSchema<Foo> jsonSchema = JSONSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema());
    assertJSONEqual(schemaJson, SCHEMA_JSON_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 
Example 9
Source Project: pulsar   File: AvroSchemaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllowNullSchema() throws JSONException {
    AvroSchema<Foo> avroSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(avroSchema.getSchemaInfo().getSchema());
    assertJSONEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 
Example 10
Source Project: xml-avro   File: DatumBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private void setFieldFromNode(Schema schema, GenericData.Record record, Node node) {
    if (node.getNodeType() != Node.ELEMENT_NODE)
        return;

    Element child = (Element) node;
    boolean setRecordFromNode = false;
    final String fieldName = child.getLocalName();
    Schema.Field field = getFieldBySource(schema, new Source(fieldName, false));
    if(field == null) {
      field = getNestedFieldBySource(schema, new Source(fieldName, false));
      setRecordFromNode = true;
    }

    if (field != null) {
        boolean array = field.schema().getType() == Schema.Type.ARRAY;
      Object datum = createNodeDatum(!array ? field.schema() : field.schema().getElementType(), child, setRecordFromNode);

        if (!array)
            record.put(field.name(), datum);
        else {
            @SuppressWarnings("unchecked") List<Object> values = (List<Object>) record.get(field.name());
            values.add(datum);
        }
    } else {
        Schema.Field anyField = schema.getField(Source.WILDCARD);
        if (anyField == null)
            throw new ConverterException("Could not find field " + fieldName + " in Avro Schema " + schema.getName() +  " , neither as specific field nor 'any' element");

        @SuppressWarnings("unchecked") Map<String, String> map = (HashMap<String, String>) record.get(Source.WILDCARD);
        map.put(fieldName, getContentAsText(child));
    }
}
 
Example 11
Source Project: xml-avro   File: ConverterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void array() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:element name='root'>" +
            "    <xs:complexType>" +
            "      <xs:sequence>" +
            "        <xs:element name='value' type='xs:string' maxOccurs='unbounded'/>" +
            "      </xs:sequence>" +
            "    </xs:complexType>" +
            "  </xs:element>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);
    Schema.Field valueField = schema.getField("value");
    assertEquals(Schema.Type.ARRAY, valueField.schema().getType());
    assertEquals(Schema.Type.STRING, valueField.schema().getElementType().getType());

    String xml = "<root>" +
                 "  <value>1</value>" +
                 "  <value>2</value>" +
                 "  <value>3</value>" +
                 "</root>";

    GenericData.Record record = Converter.createDatum(schema, xml);
    assertEquals(Arrays.asList("1", "2", "3"), record.get("value"));
}
 
Example 12
@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 
Example 13
@Test
  public void testComplexDefaultValue() throws Exception {
    Schema schema = by14.HasComplexDefaults.SCHEMA$;

    Schema.Field field = schema.getField("fieldWithDefaultEnum");
    Object specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
    Assert.assertNotNull(specificDefault);
    Assert.assertTrue(specificDefault instanceof by14.DefaultEnum);
    Object genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
    Assert.assertNotNull(genericDefault);
    Assert.assertTrue(genericDefault instanceof GenericData.EnumSymbol);

    //avro-1.4 fixed classes cant even be instantiated under avro 1.5+
    //TODO - update this test to use compat generated code
//    field = schema.getField("fieldWithDefaultFixed");
//    specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
//    Assert.assertNotNull(specificDefault);
//    Assert.assertTrue(specificDefault instanceof by14.DefaultFixed);
//    genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
//    Assert.assertNotNull(genericDefault);
//    Assert.assertTrue(genericDefault instanceof GenericData.Fixed);

    field = schema.getField("fieldWithDefaultRecord");
    specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
    Assert.assertNotNull(specificDefault);
    Assert.assertTrue(specificDefault instanceof by14.DefaultRecord);
    genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
    Assert.assertNotNull(genericDefault);
    Assert.assertTrue(genericDefault instanceof GenericData.Record);
  }
 
Example 14
Source Project: components   File: KeyValueUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generate a new Index Record which is the filtered result of the input record.
 *
 * The user can freely remove column, add empty column or change the place of column in the same hierarchical level.
 *
 * @return the new record
 */
public static IndexedRecord extractIndexedRecord(IndexedRecord inputRecord, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    Schema inputSchema = getUnwrappedSchema(inputRecord);
    for (Field field : outputSchema.getFields()) {
        if (inputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());

            // The current column can be a Record (an hierarchical sub-object) or directly a value.
            // If we are on a record, we need to recursively do the process
            // if we are on a object, we save it to the output.
            if (inputValue instanceof Record) {
                // The sub-schema at this level is a union of "empty" and a record,
                // so we need to get the true sub-schema
                Schema inputChildSchema = getUnwrappedSchema(inputSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (inputChildSchema.getType().equals(Type.RECORD)
                        && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = extractIndexedRecord((IndexedRecord) inputValue, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                }
            } else {
                outputRecord.set(field.name(), inputValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.set(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord.build();
}
 
Example 15
Source Project: nifi   File: TestJdbcCommon.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSignedIntShouldBeInt() throws SQLException, IllegalArgumentException, IllegalAccessException {
    final ResultSetMetaData metadata = mock(ResultSetMetaData.class);
    when(metadata.getColumnCount()).thenReturn(1);
    when(metadata.getColumnType(1)).thenReturn(Types.INTEGER);
    when(metadata.isSigned(1)).thenReturn(true);
    when(metadata.getColumnName(1)).thenReturn("Col1");
    when(metadata.getTableName(1)).thenReturn("Table1");

    final ResultSet rs = mock(ResultSet.class);
    when(rs.getMetaData()).thenReturn(metadata);

    Schema schema = JdbcCommon.createSchema(rs);
    Assert.assertNotNull(schema);

    Schema.Field field = schema.getField("Col1");
    Schema fieldSchema = field.schema();
    Assert.assertEquals(2, fieldSchema.getTypes().size());

    boolean foundIntSchema = false;
    boolean foundNullSchema = false;

    for (Schema type : fieldSchema.getTypes()) {
        if (type.getType().equals(Schema.Type.INT)) {
            foundIntSchema = true;
        } else if (type.getType().equals(Schema.Type.NULL)) {
            foundNullSchema = true;
        }
    }

    assertTrue(foundIntSchema);
    assertTrue(foundNullSchema);
}
 
Example 16
@Override
public void validate(Schema schema) {
  super.validate(schema);
  if (schema.getField(refFieldName) == null) {
    throw new IllegalArgumentException("Invalid field name for reference field: " + refFieldName);
  }
}
 
Example 17
Source Project: samza   File: AvroRelConverter.java    License: Apache License 2.0 5 votes vote down vote up
private static GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) {
  GenericRecord record = new GenericData.Record(schema);
  List<String> fieldNames = relRecord.getFieldNames();
  List<Object> values = relRecord.getFieldValues();
  for (int index = 0; index < fieldNames.size(); index++) {
    if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) {
      String fieldName = fieldNames.get(index);
      /**
       * It is possible that the destination Avro schema doesn't have all the fields that are projected from the
       * SQL. This is especially possible in SQL statements like
       *        insert into kafka.outputTopic select id, company from profile
       * where company is an avro record in itself whose schema can evolve. When this happens we will end up with
       * fields in the SamzaSQLRelRecord for company field which doesn't have equivalent fields in the outputTopic's schema
       * for company. To support this scenario where the input schemas and output schemas can evolve in their own cadence,
       * We ignore the fields which doesn't have corresponding schema in the output topic.
       */
      if (schema.getField(fieldName) == null) {
        LOG.debug("Schema with Name {} and Namespace {} doesn't contain the fieldName {}, Skipping it.",
            schema.getName(), schema.getNamespace(), fieldName);
        continue;
      }
      Object relObj = values.get(index);
      Schema fieldSchema = schema.getField(fieldName).schema();
      record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema)));
    }
  }

  return record;
}
 
Example 18
Source Project: kite   File: AvroEntityComposer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object buildKeyAsColumnField(String fieldName,
    Map<CharSequence, Object> keyAsColumnValues) {
  Schema schema = avroSchema.getAvroSchema();
  Field field = schema.getField(fieldName);
  if (field == null) {
    throw new ValidationException("No field named " + fieldName
        + " in schema " + schema);
  }

  Schema.Type fieldType = field.schema().getType();
  if (fieldType == Schema.Type.MAP) {
    Map<CharSequence, Object> retMap = new HashMap<CharSequence, Object>();
    for (Entry<CharSequence, Object> entry : keyAsColumnValues.entrySet()) {
      retMap.put(entry.getKey(), entry.getValue());
    }
    return retMap;
  } else if (fieldType == Schema.Type.RECORD) {
    AvroRecordBuilder<E> builder = kacRecordBuilderFactories.get(fieldName)
        .getBuilder();
    for (Entry<CharSequence, Object> keyAsColumnEntry : keyAsColumnValues
        .entrySet()) {
      builder.put(keyAsColumnEntry.getKey().toString(),
          keyAsColumnEntry.getValue());
    }
    return builder.build();
  } else {
    throw new ValidationException(
        "Only MAP or RECORD type valid for keyAsColumn fields. Found "
            + fieldType);
  }
}
 
Example 19
Source Project: kite   File: AvroUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Returns true if the types of two avro schemas are equal. This ignores
 * things like custom field properties that the equals() implementation of
 * Schema checks.
 * 
 * @param schema1
 *          The first schema to compare
 * @param schema2
 *          The second schema to compare
 * @return True if the types are equal, otherwise false.
 */
public static boolean avroSchemaTypesEqual(Schema schema1, Schema schema2) {
  if (schema1.getType() != schema2.getType()) {
    // if the types aren't equal, no need to go further. Return false
    return false;
  }

  if (schema1.getType() == Schema.Type.ENUM
      || schema1.getType() == Schema.Type.FIXED) {
    // Enum and Fixed types schemas should be equal using the Schema.equals
    // method.
    return schema1.equals(schema2);
  }
  if (schema1.getType() == Schema.Type.ARRAY) {
    // Avro element schemas should be equal, which is tested by recursively
    // calling this method.
    return avroSchemaTypesEqual(schema1.getElementType(),
        schema2.getElementType());
  } else if (schema1.getType() == Schema.Type.MAP) {
    // Map type values schemas should be equal, which is tested by recursively
    // calling this method.
    return avroSchemaTypesEqual(schema1.getValueType(),
        schema2.getValueType());
  } else if (schema1.getType() == Schema.Type.UNION) {
    // Compare Union fields in the same position by comparing their schemas
    // recursively calling this method.
    if (schema1.getTypes().size() != schema2.getTypes().size()) {
      return false;
    }
    for (int i = 0; i < schema1.getTypes().size(); i++) {
      if (!avroSchemaTypesEqual(schema1.getTypes().get(i), schema2.getTypes()
          .get(i))) {
        return false;
      }
    }
    return true;
  } else if (schema1.getType() == Schema.Type.RECORD) {
    // Compare record fields that match in name by comparing their schemas
    // recursively calling this method.
    if (schema1.getFields().size() != schema2.getFields().size()) {
      return false;
    }
    for (Field field1 : schema1.getFields()) {
      Field field2 = schema2.getField(field1.name());
      if (field2 == null) {
        return false;
      }
      if (!avroSchemaTypesEqual(field1.schema(), field2.schema())) {
        return false;
      }
    }
    return true;
  } else {
    // All other types are primitive, so them matching in type is enough.
    return true;
  }
}
 
Example 20
Source Project: spork   File: AvroStorageUtils.java    License: Apache License 2.0 4 votes vote down vote up
/** get field schema given index number */
public static Field getUDField(Schema s, int index) {
    return s.getField(getDummyFieldName(index));
}