Java Code Examples for org.apache.avro.Schema#getLogicalType()

The following examples show how to use org.apache.avro.Schema#getLogicalType() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example 2
private static <T> T visitArray(Type type, Schema array, AvroSchemaWithTypeVisitor<T> visitor) {
  if (array.getLogicalType() instanceof LogicalMap || (type != null && type.isMapType())) {
    Preconditions.checkState(
        AvroSchemaUtil.isKeyValueSchema(array.getElementType()),
        "Cannot visit invalid logical map type: %s", array);
    Types.MapType map = type != null ? type.asMapType() : null;
    List<Schema.Field> keyValueFields = array.getElementType().getFields();
    return visitor.map(map, array,
        visit(map != null ? map.keyType() : null, keyValueFields.get(0).schema(), visitor),
        visit(map != null ? map.valueType() : null, keyValueFields.get(1).schema(), visitor));

  } else {
    Types.ListType list = type != null ? type.asListType() : null;
    return visitor.array(list, array,
        visit(list != null ? list.elementType() : null, array.getElementType(), visitor));
  }
}
 
Example 3
private int convertFromTime(Schema schema, Time date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timeMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted % 86400000L);
	} else {
		throw new RuntimeException("Unsupported time type.");
	}
}
 
Example 4
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example 5
Source Project: components   File: ConvertAvroTypeToSQL.java    License: Apache License 2.0 5 votes vote down vote up
public int convertToSQLType(Schema schema) {
    Schema.Type type = schema.getType(); // The standard Avro Type
    LogicalType logicalType = schema.getLogicalType(); // The logical type for Data by example
    String javaType = schema.getProp(SchemaConstants.JAVA_CLASS_FLAG);  // And the Talend java type if standard Avro type is Union

    if (logicalType == null && type == Schema.Type.UNION) {
        for (Schema s : schema.getTypes()) {
            logicalType = null;
            if (s.getType() != Schema.Type.NULL) {
                type = s.getType();
                javaType = s.getProp(SchemaConstants.JAVA_CLASS_FLAG);
                logicalType = s.getLogicalType();
                if (javaType == null && logicalType == null) {
                    type = s.getType(); // Get Avro type if JAVA_CLASS_FLAG is not defined
                }
                break;
            }
        }
    }

    int sqlType = Types.NULL;
    if (logicalType != null) {
        sqlType = convertAvroLogicialType(logicalType);
    } else if (javaType == null) {
        sqlType = convertRawAvroType(type);
    } else {
        sqlType = convertTalendAvroType(javaType);
    }

    if(this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.containsKey(sqlType)){
        sqlType = this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.get(sqlType);
    }

    return sqlType;
}
 
Example 6
private int convertFromTime(Schema schema, Time date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timeMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted % 86400000L);
	} else {
		throw new RuntimeException("Unsupported time type.");
	}
}
 
Example 7
private long convertFromTimestamp(Schema schema, Timestamp date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timestampMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		return time + (long) LOCAL_TZ.getOffset(time);
	} else {
		throw new RuntimeException("Unsupported timestamp type.");
	}
}
 
Example 8
private int convertFromDate(Schema schema, Date date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.date()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted / 86400000L);
	} else {
		throw new RuntimeException("Unsupported date type.");
	}
}
 
Example 9
private int convertFromTime(Schema schema, Time date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timeMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted % 86400000L);
	} else {
		throw new RuntimeException("Unsupported time type.");
	}
}
 
Example 10
Source Project: parquet-mr   File: ParentValueContainer.java    License: Apache License 2.0 5 votes vote down vote up
public LogicalTypePrimitiveContainer(ParentValueContainer wrapped,
                                     Schema schema, Conversion conversion) {
  this.wrapped = wrapped;
  this.schema = schema;
  this.logicalType = schema.getLogicalType();
  this.conversion = conversion;
}
 
Example 11
Source Project: parquet-mr   File: AvroWriteSupport.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}
 
Example 12
/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}
 
Example 13
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 
Example 14
private Type convertFieldUsingLogicalType(String fieldName, Schema schema, Type.Repetition repetition) {
  LOG.debug("Converting field: {} using LogicalType", fieldName);
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();

  LogicalType logicalType = schema.getLogicalType();

  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    // Special case handling timestamp until int96 fully supported or logical types correctly supported
    if (logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros) {
      LOG.debug("Logical type is a timestamp millis or micros");
      builder = Types.primitive(INT96, repetition);
    } else {
      builder = Types.primitive(INT64, repetition);
    }
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    builder = Types.primitive(BINARY, repetition).as(UTF8);
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(ENUM);
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
                   .length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence

  if (logicalType != null &&
      !(logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros)) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      builder = builder.as(DECIMAL)
                       .precision(((LogicalTypes.Decimal) logicalType).getPrecision())
                       .scale(((LogicalTypes.Decimal) logicalType).getScale());

    } else {
      OriginalType annotation = convertLogicalType(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 15
Source Project: flink   File: AvroSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 
Example 16
Source Project: envelope   File: AvroUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Convert Avro Types into their associated DataType.
 *
 * @param schemaType Avro Schema.Type
 * @return DataType representation
 */
public static DataType dataTypeFor(Schema schemaType) {
  LOG.trace("Converting Schema[{}] to DataType", schemaType);

  // Unwrap "optional" unions to the base type
  boolean isOptional = isNullable(schemaType);

  if (isOptional) {
    // if only 2 items in the union, then "unwrap," otherwise, it's a full union and should be rendered as such
    if (schemaType.getTypes().size() == 2) {
      LOG.trace("Unwrapping simple 'optional' union for {}", schemaType);
      for (Schema s : schemaType.getTypes()) {
        if (s.getType().equals(NULL)) {
          continue;
        }
        // Unwrap
        schemaType = s;
        break;
      }
    }
  }

  // Convert supported LogicalTypes
  if (null != schemaType.getLogicalType()) {
    LogicalType logicalType = schemaType.getLogicalType();
    switch (logicalType.getName()) {
      case "date" :
        return DataTypes.DateType;
      case "timestamp-millis" :
        return DataTypes.TimestampType;
      case "decimal" :
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
        return DataTypes.createDecimalType(decimal.getPrecision(), decimal.getScale());
      default:
        // Pass-thru
        LOG.warn("Unsupported LogicalType[{}], continuing with underlying base type", logicalType.getName());
    }
  }

  switch (schemaType.getType()) {
    case RECORD:
      // StructType
      List<StructField> structFieldList = Lists.newArrayListWithCapacity(schemaType.getFields().size());
      for (Field f : schemaType.getFields()) {
        structFieldList.add(DataTypes.createStructField(f.name(), dataTypeFor(f.schema()), isNullable(f.schema())));
      }
      return DataTypes.createStructType(structFieldList);
    case ARRAY:
      Schema elementType = schemaType.getElementType();
      return DataTypes.createArrayType(dataTypeFor(elementType), isNullable(elementType));
    case MAP:
      Schema valueType = schemaType.getValueType();
      return DataTypes.createMapType(DataTypes.StringType, dataTypeFor(valueType), isNullable(valueType));
    case UNION:
      // StructType of members
      List<StructField> unionFieldList = Lists.newArrayListWithCapacity(schemaType.getTypes().size());
      int m = 0;
      for (Schema u : schemaType.getTypes()) {
        unionFieldList.add(DataTypes.createStructField("member" + m++, dataTypeFor(u), isNullable(u)));
      }
      return DataTypes.createStructType(unionFieldList);
    case FIXED:
    case BYTES:
      return DataTypes.BinaryType;
    case ENUM:
    case STRING:
      return DataTypes.StringType;
    case INT:
      return DataTypes.IntegerType;
    case LONG:
      return DataTypes.LongType;
    case FLOAT:
      return DataTypes.FloatType;
    case DOUBLE:
      return DataTypes.DoubleType;
    case BOOLEAN:
      return DataTypes.BooleanType;
    case NULL:
      return DataTypes.NullType;
    default:
      throw new RuntimeException(String.format("Unrecognized or unsupported Avro Type conversion: %s", schemaType));
  }
}
 
Example 17
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
	final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
	return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
 
Example 18
Source Project: parquet-mr   File: AvroSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
  Types.PrimitiveBuilder<PrimitiveType> builder;
  Schema.Type type = schema.getType();
  LogicalType logicalType = schema.getLogicalType();
  if (type.equals(Schema.Type.BOOLEAN)) {
    builder = Types.primitive(BOOLEAN, repetition);
  } else if (type.equals(Schema.Type.INT)) {
    builder = Types.primitive(INT32, repetition);
  } else if (type.equals(Schema.Type.LONG)) {
    builder = Types.primitive(INT64, repetition);
  } else if (type.equals(Schema.Type.FLOAT)) {
    builder = Types.primitive(FLOAT, repetition);
  } else if (type.equals(Schema.Type.DOUBLE)) {
    builder = Types.primitive(DOUBLE, repetition);
  } else if (type.equals(Schema.Type.BYTES)) {
    builder = Types.primitive(BINARY, repetition);
  } else if (type.equals(Schema.Type.STRING)) {
    if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
      builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
          .length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
    } else {
      builder = Types.primitive(BINARY, repetition).as(stringType());
    }
  } else if (type.equals(Schema.Type.RECORD)) {
    return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
  } else if (type.equals(Schema.Type.ENUM)) {
    builder = Types.primitive(BINARY, repetition).as(enumType());
  } else if (type.equals(Schema.Type.ARRAY)) {
    if (writeOldListStructure) {
      return ConversionPatterns.listType(repetition, fieldName,
          convertField("array", schema.getElementType(), REPEATED));
    } else {
      return ConversionPatterns.listOfElements(repetition, fieldName,
          convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
    }
  } else if (type.equals(Schema.Type.MAP)) {
    Type valType = convertField("value", schema.getValueType());
    // avro map key type is always string
    return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
  } else if (type.equals(Schema.Type.FIXED)) {
    builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
        .length(schema.getFixedSize());
  } else if (type.equals(Schema.Type.UNION)) {
    return convertUnion(fieldName, schema, repetition);
  } else {
    throw new UnsupportedOperationException("Cannot convert Avro type " + type);
  }

  // schema translation can only be done for known logical types because this
  // creates an equivalence
  if (logicalType != null) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
      builder = builder.as(decimalType(decimal.getScale(), decimal.getPrecision()));
    } else {
      LogicalTypeAnnotation annotation = convertLogicalType(logicalType);
      if (annotation != null) {
        builder.as(annotation);
      }
    }
  }

  return builder.named(fieldName);
}
 
Example 19
@SuppressWarnings ( "squid:S3776" )
// suppress complexity warning.  Extracting/consolidating mapping logic would make this _less_ readable.
private static AvroInputField setFieldType( Schema s, AvroInputField newField ) {
  switch ( s.getType() ) {
    case BOOLEAN:
      newField.setAvroType( AvroSpec.DataType.BOOLEAN );
      newField.setPentahoType( ValueMetaInterface.TYPE_BOOLEAN );
      break;
    case ENUM:
    case STRING:
      newField.setPentahoType( ValueMetaInterface.TYPE_STRING );
      newField.setAvroType( AvroSpec.DataType.STRING );
      break;
    case FLOAT:
      newField.setAvroType( AvroSpec.DataType.FLOAT );
      newField.setPentahoType( ValueMetaInterface.TYPE_NUMBER );
      break;
    case DOUBLE:
      newField.setAvroType( AvroSpec.DataType.DOUBLE );
      newField.setPentahoType( ValueMetaInterface.TYPE_NUMBER );
      break;
    case INT:
      if ( s.getLogicalType() != null ) {
        if ( s.getLogicalType().getName().equalsIgnoreCase( "Date" ) ) {
          newField.setAvroType( AvroSpec.DataType.DATE );
          newField.setPentahoType( ValueMetaInterface.TYPE_DATE );
        } else {
          newField.setAvroType( AvroSpec.DataType.INTEGER );
          newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
        }
      } else {
        newField.setAvroType( AvroSpec.DataType.INTEGER );
        newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
      }
      break;
    case LONG:
      if ( s.getLogicalType() != null ) {
        if ( s.getLogicalType().getName().equalsIgnoreCase( "timestamp-millis" ) ) {
          newField.setAvroType( AvroSpec.DataType.TIMESTAMP_MILLIS );
          newField.setPentahoType( ValueMetaInterface.TYPE_TIMESTAMP );
        } else {
          newField.setAvroType( AvroSpec.DataType.LONG );
          newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
        }
      } else {
        newField.setAvroType( AvroSpec.DataType.LONG );
        newField.setPentahoType( ValueMetaInterface.TYPE_INTEGER );
      }
      break;
    case BYTES:
      if ( s.getLogicalType() != null ) {
        if ( s.getLogicalType().getName().equalsIgnoreCase( "Decimal" ) ) {
          newField.setAvroType( AvroSpec.DataType.DECIMAL );
          newField.setPentahoType( ValueMetaInterface.TYPE_BIGNUMBER );
        } else {
          newField.setAvroType( AvroSpec.DataType.BYTES );
          newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
        }
      } else {
        newField.setAvroType( AvroSpec.DataType.BYTES );
        newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
      }
      break;
    case FIXED:
      newField.setAvroType( AvroSpec.DataType.FIXED );
      newField.setPentahoType( ValueMetaInterface.TYPE_BINARY );
      break;
    default:
      // unhandled type
      newField = null;
  }
  return newField;
}
 
Example 20
Source Project: flink   File: AvroSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}