Java Code Examples for org.apache.avro.Schema.Field#defaultValue()

The following examples show how to use org.apache.avro.Schema.Field#defaultValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CSVUtils.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
/**
 *
 */
private static JsonNode retrieveDefaultFieldValue(Field field) {
    JsonNode jsonNode = field.defaultValue();
    if (null == jsonNode) {
        throw new IllegalArgumentException("The field '" + field.name() + "' is NULL and there is no default value supplied in the Avro Schema");
    }
    return jsonNode;
}
Example 2
Source File: AvroUtils.java    From Cubert with Apache License 2.0 5 votes vote down vote up 
public static Schema getSchema(SeekableInput input) throws IOException
{
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
    DataFileReader<GenericRecord> dataFileReader =
            new DataFileReader<GenericRecord>(input, datumReader);
    Schema schema = dataFileReader.getSchema();

    if (PadDefaultNullsToSchema)
    {
        // a list of "cloned" fields, with optional default value set to null
        ArrayList<Field> paddedFields = new ArrayList<Field>();

        for (Field field: schema.getFields())
        {
            // should this field be padded?
            boolean needsNullPadding = (field.schema() != null) // the field has nested schema
                && (field.schema().getType().equals(Type.UNION)) // the nested schema is UNION
                && (field.schema().getTypes().get(0).getType().equals(Type.NULL)); // the first element of union is NULL type

            JsonNode defValue = needsNullPadding ? NullNode.getInstance() : field.defaultValue();

            Field f = new Field(field.name(), field.schema(), field.doc(), defValue);
            paddedFields.add(f);
        }

        schema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
        schema.setFields(paddedFields);
    }

    return schema;
}
Example 3
Source File: EnvelopePayloadConverter.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
/**
 * Convert to the output schema of a field
 */
protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit)
    throws SchemaConversionException {
  if (field.name().equals(payloadField)) {
    // Create a payload field with latest schema
    return createLatestPayloadField(field);
  }
  // Make a copy of the field to the output schema
  return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order());
}
Example 4
Source File: EnvelopePayloadConverter.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a payload field with its latest schema fetched from {@link #registry}
 *
 * @param field the original payload field from input envelope schema
 * @return a new payload field with its latest schema
 */
private Field createLatestPayloadField(Field field)
    throws SchemaConversionException {
  try {
    Schema payloadSchema = fetchLatestPayloadSchema();
    return new Field(field.name(), payloadSchema, DECORATED_PAYLOAD_DOC, field.defaultValue(), field.order());
  } catch (Exception e) {
    throw new SchemaConversionException(e);
  }
}
Example 5
Source File: PigSchema2Avro.java    From spork with Apache License 2.0 5 votes vote down vote up 
/**
 * Validate a Pig tuple is compatible with Avro record. If the Avro schema 
 * is not complete (with uncovered fields), then convert those fields using 
 * methods in set 1. 
 * 
 * Notice that users can get rid of Pig tuple wrappers, e.g. an Avro schema
 * "int" is compatible with a Pig schema "T:(int)"
 * 
 */
protected static Schema validateAndConvertRecord(Schema avroSchema, ResourceFieldSchema[] pigFields) throws IOException {

    /* Get rid of Pig tuple wrappers. */
    if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
        if (pigFields.length != 1)
            throw new IOException("Expect only one field in Pig tuple schema. Avro schema is " + avroSchema.getType());

        return validateAndConvert(avroSchema, pigFields[0]);
    }

    /* validate and convert a pig tuple with avro record */
    boolean isPartialSchema = AvroStorageUtils.isUDPartialRecordSchema(avroSchema);
    AvroStorageLog.details("isPartialSchema=" + isPartialSchema);

    String typeName = isPartialSchema ? getRecordName() : avroSchema.getName();
    Schema outSchema = Schema.createRecord(typeName, avroSchema.getDoc(), avroSchema.getNamespace(), false);

    List<Schema.Field> inFields = avroSchema.getFields();
    if (!isPartialSchema && inFields.size() != pigFields.length) {
        throw new IOException("Expect " + inFields.size() + " fields in pig schema." + " But there are " + pigFields.length);
    }

    List<Schema.Field> outFields = new ArrayList<Schema.Field>();

    for (int i = 0; i < pigFields.length; i++) {
        /* get user defined avro field schema */
        Field inputField = isPartialSchema ? AvroStorageUtils.getUDField(avroSchema, i) : inFields.get(i);

        /* get schema */
        Schema fieldSchema = null;
        if (inputField == null) { 
            /* convert pig schema (nullable) */
            fieldSchema = convert(pigFields[i], true);
        } else if (inputField.schema() == null) { 
            /* convert pig schema (not-null) */
            fieldSchema = convert(pigFields[i], false);
        } else { 
            /* validate pigFields[i] with given avro schema */
            fieldSchema = validateAndConvert(inputField.schema(),
                                            pigFields[i]);
        }

        /* get field name of output */
        String outname = (isPartialSchema) ? pigFields[i].getName() : inputField.name();
        if (outname == null)
            outname = FIELD_NAME + "_" + i; // field name cannot be null

        /* get doc of output */
        String doc = (isPartialSchema) ? pigFields[i].getDescription() : inputField.doc();

        JsonNode defaultvalue = (inputField != null) ? inputField.defaultValue() : null;

        outFields.add(new Field(outname, fieldSchema, doc, defaultvalue));

    }

    outSchema.setFields(outFields);
    return outSchema;

}
Example 6
Source File: AvroEntitySchema.java    From kite with Apache License 2.0 5 votes vote down vote up 
private static boolean fieldsEqual(Field field1, FieldMapping field1Mapping,
    Field field2, FieldMapping field2Mapping) {
  // if names aren't equal, return false
  if (!field1.name().equals(field2.name())) {
    return false;
  }
  // if schemas aren't equal, return false
  if (!AvroUtils.avroSchemaTypesEqual(field1.schema(), field2.schema())) {
    return false;
  }
  // if field mappings aren't equal, return false
  if (!Objects.equal(field1Mapping, field2Mapping)) {
    return false;
  }
  // if one default value is null and the other isn't, return false
  if ((field1.defaultValue() != null && field2.defaultValue() == null)
      || (field1.defaultValue() == null && field2.defaultValue() != null)) {
    return false;
  }
  // if both default values are not null, and the default values are not
  // equal, return false
  if ((field1.defaultValue() != null && field2.defaultValue() != null)
      && !field1.defaultValue().equals(field2.defaultValue())) {
    return false;
  }
  // Fields are equal, return true
  return true;
}
Example 7
Source File: PigSchema2Avro.java    From Cubert with Apache License 2.0 4 votes vote down vote up 
/**
 * Validate a Pig tuple is compatible with Avro record. If the Avro schema 
 * is not complete (with uncovered fields), then convert those fields using 
 * methods in set 1. 
 * 
 * Notice that users can get rid of Pig tuple wrappers, e.g. an Avro schema
 * "int" is compatible with a Pig schema "T:(int)"
 * 
 */
protected static Schema validateAndConvertRecord(Schema avroSchema, ResourceFieldSchema[] pigFields) throws IOException {

    /* Get rid of Pig tuple wrappers. */
    if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
        if (pigFields.length != 1)
            throw new IOException("Expect only one field in Pig tuple schema. Avro schema is " + avroSchema.getType());

        return validateAndConvert(avroSchema, pigFields[0]);
    }

    /* validate and convert a pig tuple with avro record */
    boolean isPartialSchema = AvroStorageUtils.isUDPartialRecordSchema(avroSchema);
    AvroStorageLog.details("isPartialSchema=" + isPartialSchema);

    String typeName = isPartialSchema ? getRecordName() : avroSchema.getName();
    Schema outSchema = Schema.createRecord(typeName, avroSchema.getDoc(), avroSchema.getNamespace(), false);

    List<Schema.Field> inFields = avroSchema.getFields();
    if (!isPartialSchema && inFields.size() != pigFields.length) {
        throw new IOException("Expect " + inFields.size() + " fields in pig schema." + " But there are " + pigFields.length);
    }

    List<Schema.Field> outFields = new ArrayList<Schema.Field>();

    for (int i = 0; i < pigFields.length; i++) {
        /* get user defined avro field schema */
        Field inputField = isPartialSchema ? AvroStorageUtils.getUDField(avroSchema, i) : inFields.get(i);

        /* get schema */
        Schema fieldSchema = null;
        if (inputField == null) { 
            /* convert pig schema (nullable) */
            fieldSchema = convert(pigFields[i], true);
        } else if (inputField.schema() == null) { 
            /* convert pig schema (not-null) */
            fieldSchema = convert(pigFields[i], false);
        } else { 
            /* validate pigFields[i] with given avro schema */
            fieldSchema = validateAndConvert(inputField.schema(),
                                            pigFields[i]);
        }

        /* get field name of output */
        String outname = (isPartialSchema) ? pigFields[i].getName() : inputField.name();
        if (outname == null)
            outname = FIELD_NAME + "_" + i; // field name cannot be null

        /* get doc of output */
        String doc = (isPartialSchema) ? pigFields[i].getDescription() : inputField.doc();

        JsonNode defaultvalue = (inputField != null) ? inputField.defaultValue() : null;

        outFields.add(new Field(outname, fieldSchema, doc, defaultvalue));

    }

    outSchema.setFields(outFields);
    return outSchema;

}
Example 8
Source File: AvroUtils.java    From kite with Apache License 2.0 2 votes vote down vote up 
/**
 * Given an avro Schema.Field instance, make a clone of it.
 * 
 * @param field
 *          The field to clone.
 * @return The cloned field.
 */
public static Field cloneField(Field field) {
  return new Field(field.name(), field.schema(), field.doc(),
      field.defaultValue());
}