Java Code Examples for org.apache.avro.Schema#getLogicalType()
The following examples show how to use
org.apache.avro.Schema#getLogicalType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: AvroSchemaWithTypeVisitor.java From iceberg with Apache License 2.0 | 6 votes |
|
private static <T> T visitArray(Type type, Schema array, AvroSchemaWithTypeVisitor<T> visitor) {
if (array.getLogicalType() instanceof LogicalMap || (type != null && type.isMapType())) {
Preconditions.checkState(
AvroSchemaUtil.isKeyValueSchema(array.getElementType()),
"Cannot visit invalid logical map type: %s", array);
Types.MapType map = type != null ? type.asMapType() : null;
List<Schema.Field> keyValueFields = array.getElementType().getFields();
return visitor.map(map, array,
visit(map != null ? map.keyType() : null, keyValueFields.get(0).schema(), visitor),
visit(map != null ? map.valueType() : null, keyValueFields.get(1).schema(), visitor));
} else {
Types.ListType list = type != null ? type.asListType() : null;
return visitor.array(list, array,
visit(list != null ? list.elementType() : null, array.getElementType(), visitor));
}
}
Example 2
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 6 votes |
|
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
// rescale to target type
final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
// byte array must contain the two's-complement representation of the
// unscaled integer value in big-endian byte order
return decimal.unscaledValue().toByteArray();
} else {
throw new RuntimeException("Unsupported decimal type.");
}
}
Example 3
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private int convertFromTime(Schema schema, Time date) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.timeMillis()) {
// adopted from Apache Calcite
final long time = date.getTime();
final long converted = time + (long) LOCAL_TZ.getOffset(time);
return (int) (converted % 86400000L);
} else {
throw new RuntimeException("Unsupported time type.");
}
}
Example 4
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
// rescale to target type
final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
// byte array must contain the two's-complement representation of the
// unscaled integer value in big-endian byte order
return decimal.unscaledValue().toByteArray();
} else {
throw new RuntimeException("Unsupported decimal type.");
}
}
Example 5
| Source File: ConvertAvroTypeToSQL.java From components with Apache License 2.0 | 5 votes |
|
public int convertToSQLType(Schema schema) {
Schema.Type type = schema.getType(); // The standard Avro Type
LogicalType logicalType = schema.getLogicalType(); // The logical type for Data by example
String javaType = schema.getProp(SchemaConstants.JAVA_CLASS_FLAG); // And the Talend java type if standard Avro type is Union
if (logicalType == null && type == Schema.Type.UNION) {
for (Schema s : schema.getTypes()) {
logicalType = null;
if (s.getType() != Schema.Type.NULL) {
type = s.getType();
javaType = s.getProp(SchemaConstants.JAVA_CLASS_FLAG);
logicalType = s.getLogicalType();
if (javaType == null && logicalType == null) {
type = s.getType(); // Get Avro type if JAVA_CLASS_FLAG is not defined
}
break;
}
}
}
int sqlType = Types.NULL;
if (logicalType != null) {
sqlType = convertAvroLogicialType(logicalType);
} else if (javaType == null) {
sqlType = convertRawAvroType(type);
} else {
sqlType = convertTalendAvroType(javaType);
}
if(this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.containsKey(sqlType)){
sqlType = this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.get(sqlType);
}
return sqlType;
}
Example 6
| Source File: AvroRowSerializationSchema.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private int convertFromTime(Schema schema, Time date) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.timeMillis()) {
// adopted from Apache Calcite
final long time = date.getTime();
final long converted = time + (long) LOCAL_TZ.getOffset(time);
return (int) (converted % 86400000L);
} else {
throw new RuntimeException("Unsupported time type.");
}
}
Example 7
| Source File: AvroRowSerializationSchema.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private long convertFromTimestamp(Schema schema, Timestamp date) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.timestampMillis()) {
// adopted from Apache Calcite
final long time = date.getTime();
return time + (long) LOCAL_TZ.getOffset(time);
} else {
throw new RuntimeException("Unsupported timestamp type.");
}
}
Example 8
| Source File: AvroWriteSupportInt96Avro18.java From datacollector with Apache License 2.0 | 5 votes |
|
/**
* Calls an appropriate write method based on the value.
* Value MUST not be null.
*
* @param type the Parquet type
* @param avroSchema the Avro schema
* @param value a non-null value to write
*/
private void writeValue(Type type, Schema avroSchema, Object value) {
Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
LogicalType logicalType = nonNullAvroSchema.getLogicalType();
if (logicalType != null) {
Conversion<?> conversion = model.getConversionByClass(
value.getClass(), logicalType);
writeValueWithoutConversion(type, nonNullAvroSchema,
convert(nonNullAvroSchema, logicalType, conversion, value));
} else {
writeValueWithoutConversion(type, nonNullAvroSchema, value);
}
}
Example 9
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private int convertFromDate(Schema schema, Date date) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
// adopted from Apache Calcite
final long time = date.getTime();
final long converted = time + (long) LOCAL_TZ.getOffset(time);
return (int) (converted / 86400000L);
} else {
throw new RuntimeException("Unsupported date type.");
}
}
Example 10
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private int convertFromTime(Schema schema, Time date) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.timeMillis()) {
// adopted from Apache Calcite
final long time = date.getTime();
final long converted = time + (long) LOCAL_TZ.getOffset(time);
return (int) (converted % 86400000L);
} else {
throw new RuntimeException("Unsupported time type.");
}
}
Example 11
| Source File: AvroWriteSupport.java From parquet-mr with Apache License 2.0 | 5 votes |
|
/**
* Calls an appropriate write method based on the value.
* Value MUST not be null.
*
* @param type the Parquet type
* @param avroSchema the Avro schema
* @param value a non-null value to write
*/
private void writeValue(Type type, Schema avroSchema, Object value) {
Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
LogicalType logicalType = nonNullAvroSchema.getLogicalType();
if (logicalType != null) {
Conversion<?> conversion = model.getConversionByClass(
value.getClass(), logicalType);
writeValueWithoutConversion(type, nonNullAvroSchema,
convert(nonNullAvroSchema, logicalType, conversion, value));
} else {
writeValueWithoutConversion(type, nonNullAvroSchema, value);
}
}
Example 12
| Source File: ParentValueContainer.java From parquet-mr with Apache License 2.0 | 5 votes |
|
public LogicalTypePrimitiveContainer(ParentValueContainer wrapped,
Schema schema, Conversion conversion) {
this.wrapped = wrapped;
this.schema = schema;
this.logicalType = schema.getLogicalType();
this.conversion = conversion;
}
Example 13
| Source File: AvroSchemaConverter.java From flink with Apache License 2.0 | 4 votes |
|
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
switch (schema.getType()) {
case RECORD:
final List<Schema.Field> fields = schema.getFields();
final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
final String[] names = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final Schema.Field field = fields.get(i);
types[i] = convertToTypeInfo(field.schema());
names[i] = field.name();
}
return Types.ROW_NAMED(names, types);
case ENUM:
return Types.STRING;
case ARRAY:
// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
case MAP:
return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
case UNION:
final Schema actualSchema;
if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(1);
} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(0);
} else if (schema.getTypes().size() == 1) {
actualSchema = schema.getTypes().get(0);
} else {
// use Kryo for serialization
return Types.GENERIC(Object.class);
}
return convertToTypeInfo(actualSchema);
case FIXED:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
// convert fixed size binary data to primitive byte arrays
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case STRING:
// convert Avro's Utf8/CharSequence to String
return Types.STRING;
case BYTES:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case INT:
// logical date and time type
final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
return Types.SQL_DATE;
} else if (logicalType == LogicalTypes.timeMillis()) {
return Types.SQL_TIME;
}
return Types.INT;
case LONG:
// logical timestamp type
if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
return Types.SQL_TIMESTAMP;
}
return Types.LONG;
case FLOAT:
return Types.FLOAT;
case DOUBLE:
return Types.DOUBLE;
case BOOLEAN:
return Types.BOOLEAN;
case NULL:
return Types.VOID;
}
throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
Example 14
| Source File: AvroNestedFieldGetter.java From pentaho-hadoop-shims with Apache License 2.0 | 4 votes |
|
@SuppressWarnings ( "squid:S3776" )
// suppress complexity warning. Extracting/consolidating mapping logic would make this _less_ readable.
private static AvroInputField setFieldType( Schema s, AvroInputField newField ) {
switch ( s.getType() ) {
case BOOLEAN:
newField.setAvroType( AvroSpec.DataType.BOOLEAN );
newField.setPentahoType( ValueMetaInterface.TYPE_BOOLEAN );
break;
case ENUM:
case STRING:
newField.setPentahoType( ValueMetaInterface.TYPE_STRING );
newField.setAvroType( AvroSpec.DataType.STRING );
break;
case FLOAT:
newField.setAvroType( AvroSpec.DataType.FLOAT );
newField.setPentahoType( ValueMetaInterface.TYPE_NUMBER );
break;
case DOUBLE:
newField.setAvroType( AvroSpec.DataType.DOUBLE );
newField.setPentahoType( ValueMetaInterface.TYPE_NUMBER );
break;
case INT:
if ( s.getLogicalType() != null ) {
if ( s.getLogicalType().getName().equalsIgnoreCase( "Date" ) ) {
newField.setAvroType( AvroSpec.DataType.DATE );
newField.setPentahoType( ValueMetaInterface.TYPE_DATE );
} else {
newField.setAvroType( AvroSpec.DataType.INTEGER );
newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
}
} else {
newField.setAvroType( AvroSpec.DataType.INTEGER );
newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
}
break;
case LONG:
if ( s.getLogicalType() != null ) {
if ( s.getLogicalType().getName().equalsIgnoreCase( "timestamp-millis" ) ) {
newField.setAvroType( AvroSpec.DataType.TIMESTAMP_MILLIS );
newField.setPentahoType( ValueMetaInterface.TYPE_TIMESTAMP );
} else {
newField.setAvroType( AvroSpec.DataType.LONG );
newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
}
} else {
newField.setAvroType( AvroSpec.DataType.LONG );
newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
}
break;
case BYTES:
if ( s.getLogicalType() != null ) {
if ( s.getLogicalType().getName().equalsIgnoreCase( "Decimal" ) ) {
newField.setAvroType( AvroSpec.DataType.DECIMAL );
newField.setPentahoType( ValueMetaInterface.TYPE_BIGNUMBER );
} else {
newField.setAvroType( AvroSpec.DataType.BYTES );
newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
}
} else {
newField.setAvroType( AvroSpec.DataType.BYTES );
newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
}
break;
case FIXED:
newField.setAvroType( AvroSpec.DataType.FIXED );
newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
break;
default:
// unhandled type
newField = null;
}
return newField;
}
Example 15
| Source File: AvroSchemaConverter.java From parquet-mr with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("deprecation")
private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
Types.PrimitiveBuilder<PrimitiveType> builder;
Schema.Type type = schema.getType();
LogicalType logicalType = schema.getLogicalType();
if (type.equals(Schema.Type.BOOLEAN)) {
builder = Types.primitive(BOOLEAN, repetition);
} else if (type.equals(Schema.Type.INT)) {
builder = Types.primitive(INT32, repetition);
} else if (type.equals(Schema.Type.LONG)) {
builder = Types.primitive(INT64, repetition);
} else if (type.equals(Schema.Type.FLOAT)) {
builder = Types.primitive(FLOAT, repetition);
} else if (type.equals(Schema.Type.DOUBLE)) {
builder = Types.primitive(DOUBLE, repetition);
} else if (type.equals(Schema.Type.BYTES)) {
builder = Types.primitive(BINARY, repetition);
} else if (type.equals(Schema.Type.STRING)) {
if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
} else {
builder = Types.primitive(BINARY, repetition).as(stringType());
}
} else if (type.equals(Schema.Type.RECORD)) {
return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
} else if (type.equals(Schema.Type.ENUM)) {
builder = Types.primitive(BINARY, repetition).as(enumType());
} else if (type.equals(Schema.Type.ARRAY)) {
if (writeOldListStructure) {
return ConversionPatterns.listType(repetition, fieldName,
convertField("array", schema.getElementType(), REPEATED));
} else {
return ConversionPatterns.listOfElements(repetition, fieldName,
convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
}
} else if (type.equals(Schema.Type.MAP)) {
Type valType = convertField("value", schema.getValueType());
// avro map key type is always string
return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
} else if (type.equals(Schema.Type.FIXED)) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(schema.getFixedSize());
} else if (type.equals(Schema.Type.UNION)) {
return convertUnion(fieldName, schema, repetition);
} else {
throw new UnsupportedOperationException("Cannot convert Avro type " + type);
}
// schema translation can only be done for known logical types because this
// creates an equivalence
if (logicalType != null) {
if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
builder = builder.as(decimalType(decimal.getScale(), decimal.getPrecision()));
} else {
LogicalTypeAnnotation annotation = convertLogicalType(logicalType);
if (annotation != null) {
builder.as(annotation);
}
}
}
return builder.named(fieldName);
}
Example 16
| Source File: AvroRowDeserializationSchema.java From flink with Apache License 2.0 | 4 votes |
|
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
Example 17
| Source File: AvroUtils.java From envelope with Apache License 2.0 | 4 votes |
|
/**
* Convert Avro Types into their associated DataType.
*
* @param schemaType Avro Schema.Type
* @return DataType representation
*/
public static DataType dataTypeFor(Schema schemaType) {
LOG.trace("Converting Schema[{}] to DataType", schemaType);
// Unwrap "optional" unions to the base type
boolean isOptional = isNullable(schemaType);
if (isOptional) {
// if only 2 items in the union, then "unwrap," otherwise, it's a full union and should be rendered as such
if (schemaType.getTypes().size() == 2) {
LOG.trace("Unwrapping simple 'optional' union for {}", schemaType);
for (Schema s : schemaType.getTypes()) {
if (s.getType().equals(NULL)) {
continue;
}
// Unwrap
schemaType = s;
break;
}
}
}
// Convert supported LogicalTypes
if (null != schemaType.getLogicalType()) {
LogicalType logicalType = schemaType.getLogicalType();
switch (logicalType.getName()) {
case "date" :
return DataTypes.DateType;
case "timestamp-millis" :
return DataTypes.TimestampType;
case "decimal" :
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale());
default:
// Pass-thru
LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName());
}
}
switch (schemaType.getType()) {
case RECORD:
// StructType
List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size());
for (Field f : schemaType.getFields()) {
structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema())));
}
return DataTypes.createStructType(structFieldList);
case ARRAY:
Schema elementType = schemaType.getElementType();
return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType));
case MAP:
Schema valueType = schemaType.getValueType();
return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType));
case UNION:
// StructType of members
List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size());
int m = 0;
for (Schema u : schemaType.getTypes()) {
unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u)));
}
return DataTypes.createStructType(unionFieldList);
case FIXED:
case BYTES:
return DataTypes.BinaryType;
case ENUM:
case STRING:
return DataTypes.StringType;
case INT:
return DataTypes.IntegerType;
case LONG:
return DataTypes.LongType;
case FLOAT:
return DataTypes.FloatType;
case DOUBLE:
return DataTypes.DoubleType;
case BOOLEAN:
return DataTypes.BooleanType;
case NULL:
return DataTypes.NullType;
default:
throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType));
}
}
Example 18
| Source File: AvroSchemaConverter.java From flink with Apache License 2.0 | 4 votes |
|
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
switch (schema.getType()) {
case RECORD:
final List<Schema.Field> fields = schema.getFields();
final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
final String[] names = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final Schema.Field field = fields.get(i);
types[i] = convertToTypeInfo(field.schema());
names[i] = field.name();
}
return Types.ROW_NAMED(names, types);
case ENUM:
return Types.STRING;
case ARRAY:
// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
case MAP:
return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
case UNION:
final Schema actualSchema;
if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(1);
} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(0);
} else if (schema.getTypes().size() == 1) {
actualSchema = schema.getTypes().get(0);
} else {
// use Kryo for serialization
return Types.GENERIC(Object.class);
}
return convertToTypeInfo(actualSchema);
case FIXED:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
// convert fixed size binary data to primitive byte arrays
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case STRING:
// convert Avro's Utf8/CharSequence to String
return Types.STRING;
case BYTES:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case INT:
// logical date and time type
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
return Types.SQL_DATE;
} else if (logicalType == LogicalTypes.timeMillis()) {
return Types.SQL_TIME;
}
return Types.INT;
case LONG:
// logical timestamp type
if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
return Types.SQL_TIMESTAMP;
}
return Types.LONG;
case FLOAT:
return Types.FLOAT;
case DOUBLE:
return Types.DOUBLE;
case BOOLEAN:
return Types.BOOLEAN;
case NULL:
return Types.VOID;
}
throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
Example 19
| Source File: AvroSchemaConverter190Int96Avro18.java From datacollector with Apache License 2.0 | 4 votes |
|
private Type convertFieldUsingLogicalType(String fieldName, Schema schema, Type.Repetition repetition) {
LOG.debug("Converting field: {} using LogicalType", fieldName);
Types.PrimitiveBuilder<PrimitiveType> builder;
Schema.Type type = schema.getType();
LogicalType logicalType = schema.getLogicalType();
if (type.equals(Schema.Type.BOOLEAN)) {
builder = Types.primitive(BOOLEAN, repetition);
} else if (type.equals(Schema.Type.INT)) {
builder = Types.primitive(INT32, repetition);
} else if (type.equals(Schema.Type.LONG)) {
// Special case handling timestamp until int96 fully supported or logical types correctly supported
if (logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros) {
LOG.debug("Logical type is a timestamp millis or micros");
builder = Types.primitive(INT96, repetition);
} else {
builder = Types.primitive(INT64, repetition);
}
} else if (type.equals(Schema.Type.FLOAT)) {
builder = Types.primitive(FLOAT, repetition);
} else if (type.equals(Schema.Type.DOUBLE)) {
builder = Types.primitive(DOUBLE, repetition);
} else if (type.equals(Schema.Type.BYTES)) {
builder = Types.primitive(BINARY, repetition);
} else if (type.equals(Schema.Type.STRING)) {
builder = Types.primitive(BINARY, repetition).as(UTF8);
} else if (type.equals(Schema.Type.RECORD)) {
return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
} else if (type.equals(Schema.Type.ENUM)) {
builder = Types.primitive(BINARY, repetition).as(ENUM);
} else if (type.equals(Schema.Type.ARRAY)) {
if (writeOldListStructure) {
return ConversionPatterns.listType(repetition, fieldName,
convertField("array", schema.getElementType(), REPEATED));
} else {
return ConversionPatterns.listOfElements(repetition, fieldName,
convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
}
} else if (type.equals(Schema.Type.MAP)) {
Type valType = convertField("value", schema.getValueType());
// avro map key type is always string
return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
} else if (type.equals(Schema.Type.FIXED)) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(schema.getFixedSize());
} else if (type.equals(Schema.Type.UNION)) {
return convertUnion(fieldName, schema, repetition);
} else {
throw new UnsupportedOperationException("Cannot convert Avro type " + type);
}
// schema translation can only be done for known logical types because this
// creates an equivalence
if (logicalType != null &&
!(logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros)) {
if (logicalType instanceof LogicalTypes.Decimal) {
builder = builder.as(DECIMAL)
.precision(((LogicalTypes.Decimal) logicalType).getPrecision())
.scale(((LogicalTypes.Decimal) logicalType).getScale());
} else {
OriginalType annotation = convertLogicalType(logicalType);
if (annotation != null) {
builder.as(annotation);
}
}
}
return builder.named(fieldName);
}
Example 20
| Source File: AvroSchemaConverter.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
switch (schema.getType()) {
case RECORD:
final List<Schema.Field> fields = schema.getFields();
final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
final String[] names = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final Schema.Field field = fields.get(i);
types[i] = convertToTypeInfo(field.schema());
names[i] = field.name();
}
return Types.ROW_NAMED(names, types);
case ENUM:
return Types.STRING;
case ARRAY:
// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
case MAP:
return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
case UNION:
final Schema actualSchema;
if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(1);
} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(0);
} else if (schema.getTypes().size() == 1) {
actualSchema = schema.getTypes().get(0);
} else {
// use Kryo for serialization
return Types.GENERIC(Object.class);
}
return convertToTypeInfo(actualSchema);
case FIXED:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
// convert fixed size binary data to primitive byte arrays
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case STRING:
// convert Avro's Utf8/CharSequence to String
return Types.STRING;
case BYTES:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case INT:
// logical date and time type
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
return Types.SQL_DATE;
} else if (logicalType == LogicalTypes.timeMillis()) {
return Types.SQL_TIME;
}
return Types.INT;
case LONG:
// logical timestamp type
if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
return Types.SQL_TIMESTAMP;
}
return Types.LONG;
case FLOAT:
return Types.FLOAT;
case DOUBLE:
return Types.DOUBLE;
case BOOLEAN:
return Types.BOOLEAN;
case NULL:
return Types.VOID;
}
throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
