Java Code Examples for org.apache.avro.Schema#getProp()

The following examples show how to use org.apache.avro.Schema#getProp() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Infer String class for the passed schema.
 * If the current Avro lib is avro-1.4, this function will return default String type directly.
 * Otherwise, according to Avro specification, this function only supports the following property to specify
 * Stringable class while using Specific deserializer:
 * 1. {@link Schema.Type#STRING} : {@link #CLASS_PROP};
 * 2. {@link Schema.Type#MAP} :  {@link #KEY_CLASS_PROP};
 *
 * If the above property is absent for Specific deserializer or the current deserializer is generic,
 * this function will try to look at {@link #STRING_PROP} to decide whether it should use {@link String}
 * or the default String type.
 * @param schema
 * @return
 */
public JClass findStringClass(Schema schema) {
  Schema.Type schemaType = schema.getType();
  if (!schemaType.equals(Schema.Type.STRING) && !schemaType.equals(Schema.Type.MAP)) {
    throw new SchemaAssistantException("Function `findStringClass` only supports schema types: " + Schema.Type.STRING + " and " + Schema.Type.MAP);
  }
  JClass outputClass = defaultStringType();
  if (!Utils.isAvro14()) {
    String stringClassProp;
    if (schemaType.equals(Schema.Type.STRING)) {
      stringClassProp = schema.getProp(CLASS_PROP);
    } else {
      stringClassProp = schema.getProp(KEY_CLASS_PROP);
    }
    if (!useGenericTypes && null != stringClassProp) {
        outputClass = codeModel.ref(stringClassProp);
        extendExceptionsFromStringable(schema.getProp(CLASS_PROP));
    } else {
      String stringProp = schema.getProp(STRING_PROP);
      if (null != stringProp && stringProp.equals(STRING_TYPE_STRING)) {
        outputClass = codeModel.ref(String.class);
      }
    }
  }
  return outputClass;
}
 
Example 2
private Schema initDefaultSchema(Schema designSchema) {
    AvroRegistry avroReg = new AvroRegistry();
    FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields();
    for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) {
        Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema();
        FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName());
        if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) {
            String datePattern;
            Field designField = designSchema.getField(metadataField.getName());
            if (designField != null) {
                datePattern = designField.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            } else {
                datePattern = designSchema.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            }
            if (datePattern == null || datePattern.isEmpty()) {
                datePattern = "dd-MM-yyyy";
            }
            fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, datePattern);
        }
        fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault();
    }
    Schema defaultSchema = record.endRecord();
    return defaultSchema;
}
 
Example 3
public static boolean isStringable(Schema schema) {
  if (!Schema.Type.STRING.equals(schema.getType())) {
    throw new SchemaAssistantException("String schema expected!");
  }

  if (Utils.isAvro14()) {
    return false;
  } else {
    return schema.getProp(CLASS_PROP) != null;
  }
}
 
Example 4
public static boolean hasStringableKey(Schema schema) {
  if (!Schema.Type.MAP.equals(schema.getType())) {
    throw new SchemaAssistantException("Map schema expected!");
  }

  if (Utils.isAvro14()) {
    return false;
  } else {
    return schema.getProp(KEY_CLASS_PROP) != null;
  }
}
 
Example 5
Source Project: components   File: JDBCAvroRegistry.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean isObject(Schema schema) {
    if(schema == null) {
        return false;
    }

    if(schema.getType() == Schema.Type.STRING) {
        if(schema.getProp(SchemaConstants.JAVA_CLASS_FLAG) != null) {
            return true;
        }
    }

    return false;
}
 
Example 6
Source Project: components   File: ConvertAvroTypeToSQL.java    License: Apache License 2.0 5 votes vote down vote up
public int convertToSQLType(Schema schema) {
    Schema.Type type = schema.getType(); // The standard Avro Type
    LogicalType logicalType = schema.getLogicalType(); // The logical type for Data by example
    String javaType = schema.getProp(SchemaConstants.JAVA_CLASS_FLAG);  // And the Talend java type if standard Avro type is Union

    if (logicalType == null && type == Schema.Type.UNION) {
        for (Schema s : schema.getTypes()) {
            logicalType = null;
            if (s.getType() != Schema.Type.NULL) {
                type = s.getType();
                javaType = s.getProp(SchemaConstants.JAVA_CLASS_FLAG);
                logicalType = s.getLogicalType();
                if (javaType == null && logicalType == null) {
                    type = s.getType(); // Get Avro type if JAVA_CLASS_FLAG is not defined
                }
                break;
            }
        }
    }

    int sqlType = Types.NULL;
    if (logicalType != null) {
        sqlType = convertAvroLogicialType(logicalType);
    } else if (javaType == null) {
        sqlType = convertRawAvroType(type);
    } else {
        sqlType = convertTalendAvroType(javaType);
    }

    if(this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.containsKey(sqlType)){
        sqlType = this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.get(sqlType);
    }

    return sqlType;
}
 
Example 7
Source Project: components   File: TJDBCInputProperties.java    License: Apache License 2.0 5 votes vote down vote up
private void updateTrimTable() {
    Schema schema = main.schema.getValue();
    if (schema == null) {
        return;
    }

    boolean dynamic = AvroUtils.isIncludeAllFields(schema);
    int dynamicIndex = -1;
    String dynamicFieldName = schema.getProp(ComponentConstants.TALEND6_DYNAMIC_COLUMN_NAME);
    if (dynamic) {
        dynamicIndex = Integer.valueOf(schema.getProp(ComponentConstants.TALEND6_DYNAMIC_COLUMN_POSITION));
    }

    List<String> fieldNames = new ArrayList<>();
    int i = 0;
    List<Field> fields = schema.getFields();
    for (Schema.Field f : fields) {
        if (i == dynamicIndex) {
            fieldNames.add(dynamicFieldName);
        }
        fieldNames.add(f.name());
        i++;
    }

    if (dynamicIndex == i) {
        fieldNames.add(dynamicFieldName);
    }

    trimTable.columnName.setValue(fieldNames);
}
 
Example 8
Source Project: parquet-mr   File: AvroRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
private static Class<?> getStringableClass(Schema schema, GenericData model) {
  if (model instanceof SpecificData) {
    // both specific and reflect (and any subclasses) use this logic
    boolean isMap = (schema.getType() == Schema.Type.MAP);
    String stringableClass = schema.getProp(
        isMap ? JAVA_KEY_CLASS_PROP : JAVA_CLASS_PROP);
    if (stringableClass != null) {
      try {
        return ClassUtils.forName(model.getClassLoader(), stringableClass);
      } catch (ClassNotFoundException e) {
        // not available, use a String instead
      }
    }
  }

  if (ReflectData.class.isAssignableFrom(model.getClass())) {
    // reflect uses String, not Utf8
    return String.class;
  }

  // generic and specific use the avro.java.string setting
  String name = schema.getProp(STRINGABLE_PROP);
  if (name == null) {
    return CharSequence.class;
  }

  switch (GenericData.StringType.valueOf(name)) {
    case String:
      return String.class;
    default:
      return CharSequence.class; // will use Utf8
  }
}
 
Example 9
Source Project: nifi   File: NonCachingDatumReader.java    License: Apache License 2.0 5 votes vote down vote up
protected Class findStringClass(Schema schema) {
    final String name = schema.getProp(GenericData.STRING_PROP);
    if ("String".equals(name)) {
        return String.class;
    }

    return CharSequence.class;
}
 
Example 10
default String readProp(final Schema schema, final String name) {
    return schema.getProp("talend.component." + name);
}
 
Example 11
Source Project: avro-fastserde   File: SchemaAssistant.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean isStringable(Schema schema) {
    if (!Schema.Type.STRING.equals(schema.getType())) {
        throw new SchemaAssistantException("String schema expected!");
    }
    return schema.getProp(SpecificData.CLASS_PROP) != null;
}
 
Example 12
Source Project: avro-fastserde   File: SchemaAssistant.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean hasStringableKey(Schema schema) {
    if (!Schema.Type.MAP.equals(schema.getType())) {
        throw new SchemaAssistantException("Map schema expected!");
    }
    return schema.getProp(SpecificData.KEY_CLASS_PROP) != null;
}
 
Example 13
Source Project: datacollector   File: AvroTypeUtil.java    License: Apache License 2.0 4 votes vote down vote up
private static Field.Type getFieldType(Schema schema) {
  String logicalType = schema.getProp(LOGICAL_TYPE);
  if(logicalType != null && !logicalType.isEmpty()) {
    switch (logicalType) {
      case LOGICAL_TYPE_DECIMAL:
        return Field.Type.DECIMAL;
      case LOGICAL_TYPE_DATE:
        return Field.Type.DATE;
      case LOGICAL_TYPE_TIME_MILLIS:
        return Field.Type.TIME;
      case LOGICAL_TYPE_TIME_MICROS:
        return Field.Type.LONG;
      case LOGICAL_TYPE_TIMESTAMP_MILLIS:
        return Field.Type.DATETIME;
      case LOGICAL_TYPE_TIMESTAMP_MICROS:
        return Field.Type.LONG;
    }
  }

  switch(schema.getType()) {
    case ARRAY:
      return Field.Type.LIST;
    case BOOLEAN:
      return Field.Type.BOOLEAN;
    case BYTES:
      return Field.Type.BYTE_ARRAY;
    case DOUBLE:
      return Field.Type.DOUBLE;
    case ENUM:
      return Field.Type.STRING;
    case FIXED:
      return Field.Type.BYTE_ARRAY;
    case FLOAT:
      return Field.Type.FLOAT;
    case INT:
      return Field.Type.INTEGER;
    case LONG:
      return Field.Type.LONG;
    case MAP:
      return Field.Type.MAP;
    case NULL:
      return Field.Type.MAP;
    case RECORD:
      return Field.Type.MAP;
    case STRING:
      return Field.Type.STRING;
    default:
      throw new IllegalStateException(Utils.format("Unexpected schema type {}", schema.getType().getName()));
  }
}
 
Example 14
private Type convertFieldWithoutUsingLogicalType(String fieldName, Schema schema, Type.Repetition repetition) {
  LOG.debug("Converting field: {} without using LogicalType", fieldName);
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();

  String logicalType = schema.getProp(AvroTypeUtil.LOGICAL_TYPE);

  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    // Special case handling timestamp until int96 fully supported or logical types correctly supported
    if (AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS.equals(logicalType) ||
        AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MICROS.equals(logicalType)) {
      LOG.debug("Logical type is a timestamp millis or micros");
      builder = Types.primitive(INT96, repetition);
    } else {
      builder = Types.primitive(INT64, repetition);
    }
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    builder = Types.primitive(BINARY, repetition).as(UTF8);
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(ENUM);
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
                   .length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence

  if (logicalType != null &&
      !(AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS.equals(logicalType) || AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MICROS.equals(logicalType))) {
    if (AvroTypeUtil.LOGICAL_TYPE_DECIMAL.equals(logicalType)) {
      builder = (Types.PrimitiveBuilder<PrimitiveType>) builder.as(DECIMAL)
         .precision(schema.getJsonProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION).getIntValue())
         .scale(schema.getJsonProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE).getIntValue());

    } else {
      OriginalType annotation = convertLogicalTypeStr(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 15
@SuppressWarnings("deprecation")
private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();
  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    builder = Types.primitive(INT64, repetition);
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    builder = Types.primitive(BINARY, repetition).as(UTF8);
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(ENUM);
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = (Types.PrimitiveBuilder<PrimitiveType>) Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition).length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence
  String logicalType = schema.getProp(LOGICAL_TYPE);
  if (logicalType != null) {
    if (LOGICAL_TYPE_DECIMAL.equals(logicalType)) {
      builder = (Types.PrimitiveBuilder<PrimitiveType>) builder.as(DECIMAL)
          .precision(schema.getJsonProp(LOGICAL_PROP_PRECISION).getIntValue())
          .scale(schema.getJsonProp(LOGICAL_PROP_SCALE).getIntValue());

    } else {
      OriginalType annotation = convertLogicalType(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 16
Source Project: components   File: GenericAvroRegistry.java    License: Apache License 2.0 4 votes vote down vote up
DateToStringConvert(Schema schema) {
    super(schema);
    String pattern = schema.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
    // TODO: null handling
    format = new SimpleDateFormat(pattern);
}
 
Example 17
Source Project: incubator-gobblin   File: AvroUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema) {
  if (inputSchema.getProp(SCHEMA_CREATION_TIME_KEY) != null && outputSchema.getProp(SCHEMA_CREATION_TIME_KEY) == null) {
    outputSchema.addProp(SCHEMA_CREATION_TIME_KEY, inputSchema.getProp(SCHEMA_CREATION_TIME_KEY));
  }
  return outputSchema;
}
 
Example 18
Source Project: incubator-gobblin   File: AvroUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static String getSchemaCreationTime(Schema inputSchema) {
  return inputSchema.getProp(SCHEMA_CREATION_TIME_KEY);
}
 
Example 19
public Reference(Schema schema) {
  super(REFERENCE);
  this.refFieldName = schema.getProp(REF_FIELD_NAME);
}
 
Example 20
public Referenceable(Schema schema) {
  super(REFERENCEABLE);
  this.idFieldName = schema.getProp(ID_FIELD_NAME);
}