Java Code Examples for org.apache.avro.LogicalTypes#Decimal
The following examples show how to use
org.apache.avro.LogicalTypes#Decimal .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 7 votes |
|
@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
if (logicalType == null) {
return null;
}
if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
if (decimal.getPrecision() <= 9) {
return (Conversion<Object>) intDecimalConversion;
} else if (decimal.getPrecision() <= 18) {
return (Conversion<Object>) longDecimalConversion;
} else {
return (Conversion<Object>) fixedDecimalConversion;
}
} else if ("uuid".equals(logicalType.getName())) {
return (Conversion<Object>) uuidConversion;
}
return super.getConversionFor(logicalType);
}
Example 2
| Source File: TestAvroTypeUtil.java From nifi with Apache License 2.0 | 6 votes |
|
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfBinaryAndLogicalTypeDecimal() {
// Create a field schema like {"type":"binary","name":"amount","logicalType":"decimal","precision":18,"scale":8}
final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
final Schema fieldSchema = Schema.create(Type.BYTES);
decimalType.addToSchema(fieldSchema);
// Create a field named "amount" using the field schema above
final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);
// Create an overall record schema with the amount field
final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));
// Create an example Avro record with the amount field of type binary and a logical type of decimal
final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
final GenericRecord genericRecord = new GenericData.Record(avroSchema);
genericRecord.put("amount", new Conversions.DecimalConversion().toBytes(expectedBigDecimal, fieldSchema, decimalType));
// Convert the Avro schema to a Record schema
thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
Example 3
| Source File: AvroSchemaConverter.java From parquet-mr with Apache License 2.0 | 6 votes |
|
private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) {
if (logicalType == null) {
return null;
} else if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
return decimalType(decimal.getScale(), decimal.getPrecision());
} else if (logicalType instanceof LogicalTypes.Date) {
return dateType();
} else if (logicalType instanceof LogicalTypes.TimeMillis) {
return timeType(true, MILLIS);
} else if (logicalType instanceof LogicalTypes.TimeMicros) {
return timeType(true, MICROS);
} else if (logicalType instanceof LogicalTypes.TimestampMillis) {
return timestampType(true, MILLIS);
} else if (logicalType instanceof LogicalTypes.TimestampMicros) {
return timestampType(true, MICROS);
} else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
return uuidType();
}
return null;
}
Example 4
| Source File: TestAvroTypeUtil.java From nifi with Apache License 2.0 | 6 votes |
|
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfFixedAndLogicalTypeDecimal() {
// Create a field schema like {"type":"fixed","name":"amount","size":16,"logicalType":"decimal","precision":18,"scale":8}
final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
final Schema fieldSchema = Schema.createFixed("amount", null, null, 16);
decimalType.addToSchema(fieldSchema);
// Create a field named "amount" using the field schema above
final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);
// Create an overall record schema with the amount field
final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));
// Create an example Avro record with the amount field of type fixed and a logical type of decimal
final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
final GenericRecord genericRecord = new GenericData.Record(avroSchema);
genericRecord.put("amount", new Conversions.DecimalConversion().toFixed(expectedBigDecimal, fieldSchema, decimalType));
// Convert the Avro schema to a Record schema
thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
Example 5
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 6 votes |
|
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
// rescale to target type
final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
// byte array must contain the two's-complement representation of the
// unscaled integer value in big-endian byte order
return decimal.unscaledValue().toByteArray();
} else {
throw new RuntimeException("Unsupported decimal type.");
}
}
Example 6
| Source File: TestAvroTypeUtil.java From nifi with Apache License 2.0 | 6 votes |
|
@Test
public void testFixedDecimalConversion(){
final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
final Schema fieldSchema = Schema.createFixed("mydecimal", "no doc", "myspace", 18);
decimalType.addToSchema(fieldSchema);
final Object convertedValue = AvroTypeUtil.convertToAvroObject("2.5", fieldSchema, StandardCharsets.UTF_8);
assertTrue(convertedValue instanceof GenericFixed);
final GenericFixed genericFixed = (GenericFixed)convertedValue;
final BigDecimal bigDecimal = new Conversions.DecimalConversion().fromFixed(genericFixed, fieldSchema, decimalType);
assertEquals(new BigDecimal("2.5").setScale(8), bigDecimal);
}
Example 7
| Source File: AvroNestedReader.java From pentaho-hadoop-shims with Apache License 2.0 | 5 votes |
|
/**
* @param pentahoType
* @param avroData
* @param fieldSchema
* @return
*/
public Object convertToKettleValue( AvroInputField pentahoType, ByteBuffer avroData, Schema fieldSchema ) {
Object pentahoData = null;
if ( avroData != null ) {
try {
switch ( pentahoType.getPentahoType() ) {
case ValueMetaInterface.TYPE_BIGNUMBER:
Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
Schema schema = fieldSchema;
if ( schema.getType().equals( Schema.Type.UNION ) ) {
List<Schema> schemas = schema.getTypes();
for ( Schema s : schemas ) {
if ( !s.getName().equalsIgnoreCase( "null" ) ) {
schema = s;
break;
}
}
}
Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
LogicalTypes.Decimal decimalType =
LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
pentahoData = converter.fromBytes( avroData, m_schemaToUse, decimalType );
break;
case ValueMetaInterface.TYPE_BINARY:
pentahoData = new byte[ avroData.remaining() ];
avroData.get( (byte[]) pentahoData );
break;
}
} catch ( Exception e ) {
// If unable to do the type conversion just ignore. null will be returned.
}
}
return pentahoData;
}
Example 8
| Source File: AvroRowSerializationSchema.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
// rescale to target type
final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
// byte array must contain the two's-complement representation of the
// unscaled integer value in big-endian byte order
return decimal.unscaledValue().toByteArray();
} else {
throw new RuntimeException("Unsupported decimal type.");
}
}
Example 9
| Source File: PentahoAvroRecordReader.java From pentaho-hadoop-shims with Apache License 2.0 | 5 votes |
|
private Object convertToPentahoType( int pentahoType, ByteBuffer avroData, Schema.Field field ) {
Object pentahoData = null;
if ( avroData != null ) {
try {
switch ( pentahoType ) {
case ValueMetaInterface.TYPE_BIGNUMBER:
Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
Schema schema = field.schema();
if ( schema.getType().equals( Schema.Type.UNION ) ) {
List<Schema> schemas = field.schema().getTypes();
for ( Schema s : schemas ) {
if ( !s.getName().equalsIgnoreCase( "null" ) ) {
schema = s;
break;
}
}
}
Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
LogicalTypes.Decimal decimalType =
LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
pentahoData = converter.fromBytes( avroData, avroSchema, decimalType );
break;
case ValueMetaInterface.TYPE_BINARY:
pentahoData = new byte[ avroData.remaining() ];
avroData.get( (byte[]) pentahoData );
break;
}
} catch ( Exception e ) {
// If unable to do the type conversion just ignore. null will be returned.
}
}
return pentahoData;
}
Example 10
| Source File: ConvertAvroTypeToSQL.java From components with Apache License 2.0 | 5 votes |
|
private int convertAvroLogicialType(LogicalType logicalType) {
Integer sqlType = this.config.CONVERT_LOGICALTYPE_TO_SQLTYPE.get(logicalType);
if(sqlType != null){
return sqlType;
}
if (logicalType == LogicalTypes.timestampMillis()) {
sqlType = Types.TIMESTAMP;
} else if (logicalType instanceof LogicalTypes.Decimal) {
sqlType = Types.NUMERIC;
} else if (logicalType == LogicalTypes.date()) {
sqlType = Types.DATE;
} else if (logicalType == LogicalTypes.uuid()) {
sqlType = Types.VARCHAR;
} else if (logicalType == LogicalTypes.timestampMicros()) {
sqlType = Types.TIMESTAMP;
} else if (logicalType == LogicalTypes.timeMillis()) {
sqlType = Types.TIME;
} else if (logicalType == LogicalTypes.timeMicros()) {
sqlType = Types.TIME;
} else {
// All logical type should be supported
throw new UnsupportedOperationException("Logical type " + logicalType + " not supported");
}
return sqlType;
}
Example 11
| Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
|
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
final LogicalType logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
// rescale to target type
final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
// byte array must contain the two's-complement representation of the
// unscaled integer value in big-endian byte order
return decimal.unscaledValue().toByteArray();
} else {
throw new RuntimeException("Unsupported decimal type.");
}
}
Example 12
| Source File: AvroRowDeserializationSchema.java From flink with Apache License 2.0 | 4 votes |
|
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
Example 13
| Source File: HoodieRealtimeRecordReaderUtils.java From hudi with Apache License 2.0 | 4 votes |
|
/**
* Convert the projected read from delta record into an array writable.
*/
public static Writable avroToArrayWritable(Object value, Schema schema) {
if (value == null) {
return null;
}
switch (schema.getType()) {
case STRING:
return new Text(value.toString());
case BYTES:
return new BytesWritable(((ByteBuffer)value).array());
case INT:
return new IntWritable((Integer) value);
case LONG:
return new LongWritable((Long) value);
case FLOAT:
return new FloatWritable((Float) value);
case DOUBLE:
return new DoubleWritable((Double) value);
case BOOLEAN:
return new BooleanWritable((Boolean) value);
case NULL:
return null;
case RECORD:
GenericRecord record = (GenericRecord) value;
Writable[] recordValues = new Writable[schema.getFields().size()];
int recordValueIndex = 0;
for (Schema.Field field : schema.getFields()) {
recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
}
return new ArrayWritable(Writable.class, recordValues);
case ENUM:
return new Text(value.toString());
case ARRAY:
GenericArray arrayValue = (GenericArray) value;
Writable[] arrayValues = new Writable[arrayValue.size()];
int arrayValueIndex = 0;
for (Object obj : arrayValue) {
arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
}
// Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
return new ArrayWritable(Writable.class, arrayValues);
case MAP:
Map mapValue = (Map) value;
Writable[] mapValues = new Writable[mapValue.size()];
int mapValueIndex = 0;
for (Object entry : mapValue.entrySet()) {
Map.Entry mapEntry = (Map.Entry) entry;
Writable[] nestedMapValues = new Writable[2];
nestedMapValues[0] = new Text(mapEntry.getKey().toString());
nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
}
// Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
return new ArrayWritable(Writable.class, mapValues);
case UNION:
List<Schema> types = schema.getTypes();
if (types.size() != 2) {
throw new IllegalArgumentException("Only support union with 2 fields");
}
Schema s1 = types.get(0);
Schema s2 = types.get(1);
if (s1.getType() == Schema.Type.NULL) {
return avroToArrayWritable(value, s2);
} else if (s2.getType() == Schema.Type.NULL) {
return avroToArrayWritable(value, s1);
} else {
throw new IllegalArgumentException("Only support union with null");
}
case FIXED:
if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema);
HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(),
decimal.getScale());
return HiveDecimalWritable.enforcePrecisionScale(writable,
decimal.getPrecision(),
decimal.getScale());
}
return new BytesWritable(((GenericFixed) value).bytes());
default:
return null;
}
}
Example 14
| Source File: TestJdbcCommon.java From nifi with Apache License 2.0 | 4 votes |
|
private void testConvertToAvroStreamForBigDecimal(BigDecimal bigDecimal, int dbPrecision, int defaultPrecision, int expectedPrecision, int expectedScale) throws SQLException, IOException {
final ResultSetMetaData metadata = mock(ResultSetMetaData.class);
when(metadata.getColumnCount()).thenReturn(1);
when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC);
when(metadata.getColumnName(1)).thenReturn("The.Chairman");
when(metadata.getTableName(1)).thenReturn("1the::table");
when(metadata.getPrecision(1)).thenReturn(dbPrecision);
when(metadata.getScale(1)).thenReturn(expectedScale);
final ResultSet rs = JdbcCommonTestUtils.resultSetReturningMetadata(metadata);
when(rs.getObject(Mockito.anyInt())).thenReturn(bigDecimal);
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions
.builder().convertNames(true).useLogicalTypes(true).defaultPrecision(defaultPrecision).build();
JdbcCommon.convertToAvroStream(rs, baos, options, null);
final byte[] serializedBytes = baos.toByteArray();
final InputStream instream = new ByteArrayInputStream(serializedBytes);
final GenericData genericData = new GenericData();
genericData.addLogicalTypeConversion(new Conversions.DecimalConversion());
final DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(null, null, genericData);
try (final DataFileStream<GenericRecord> dataFileReader = new DataFileStream<>(instream, datumReader)) {
final Schema generatedUnion = dataFileReader.getSchema().getField("The_Chairman").schema();
// null and decimal.
assertEquals(2, generatedUnion.getTypes().size());
final LogicalType logicalType = generatedUnion.getTypes().get(1).getLogicalType();
assertNotNull(logicalType);
assertEquals("decimal", logicalType.getName());
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
assertEquals(expectedPrecision, decimalType.getPrecision());
assertEquals(expectedScale, decimalType.getScale());
GenericRecord record = null;
while (dataFileReader.hasNext()) {
record = dataFileReader.next(record);
assertEquals("_1the__table", record.getSchema().getName());
assertEquals(bigDecimal, record.get("The_Chairman"));
}
}
}
Example 15
| Source File: AvroSchemaConverter.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
switch (schema.getType()) {
case RECORD:
final List<Schema.Field> fields = schema.getFields();
final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
final String[] names = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final Schema.Field field = fields.get(i);
types[i] = convertToTypeInfo(field.schema());
names[i] = field.name();
}
return Types.ROW_NAMED(names, types);
case ENUM:
return Types.STRING;
case ARRAY:
// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
case MAP:
return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
case UNION:
final Schema actualSchema;
if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(1);
} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(0);
} else if (schema.getTypes().size() == 1) {
actualSchema = schema.getTypes().get(0);
} else {
// use Kryo for serialization
return Types.GENERIC(Object.class);
}
return convertToTypeInfo(actualSchema);
case FIXED:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
// convert fixed size binary data to primitive byte arrays
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case STRING:
// convert Avro's Utf8/CharSequence to String
return Types.STRING;
case BYTES:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case INT:
// logical date and time type
final LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
return Types.SQL_DATE;
} else if (logicalType == LogicalTypes.timeMillis()) {
return Types.SQL_TIME;
}
return Types.INT;
case LONG:
// logical timestamp type
if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
return Types.SQL_TIMESTAMP;
}
return Types.LONG;
case FLOAT:
return Types.FLOAT;
case DOUBLE:
return Types.DOUBLE;
case BOOLEAN:
return Types.BOOLEAN;
case NULL:
return Types.VOID;
}
throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
Example 16
| Source File: AvroSchemaConverter190Int96Avro18.java From datacollector with Apache License 2.0 | 4 votes |
|
private Type convertFieldUsingLogicalType(String fieldName, Schema schema, Type.Repetition repetition) {
LOG.debug("Converting field: {} using LogicalType", fieldName);
Types.PrimitiveBuilder<PrimitiveType> builder;
Schema.Type type = schema.getType();
LogicalType logicalType = schema.getLogicalType();
if (type.equals(Schema.Type.BOOLEAN)) {
builder = Types.primitive(BOOLEAN, repetition);
} else if (type.equals(Schema.Type.INT)) {
builder = Types.primitive(INT32, repetition);
} else if (type.equals(Schema.Type.LONG)) {
// Special case handling timestamp until int96 fully supported or logical types correctly supported
if (logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros) {
LOG.debug("Logical type is a timestamp millis or micros");
builder = Types.primitive(INT96, repetition);
} else {
builder = Types.primitive(INT64, repetition);
}
} else if (type.equals(Schema.Type.FLOAT)) {
builder = Types.primitive(FLOAT, repetition);
} else if (type.equals(Schema.Type.DOUBLE)) {
builder = Types.primitive(DOUBLE, repetition);
} else if (type.equals(Schema.Type.BYTES)) {
builder = Types.primitive(BINARY, repetition);
} else if (type.equals(Schema.Type.STRING)) {
builder = Types.primitive(BINARY, repetition).as(UTF8);
} else if (type.equals(Schema.Type.RECORD)) {
return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
} else if (type.equals(Schema.Type.ENUM)) {
builder = Types.primitive(BINARY, repetition).as(ENUM);
} else if (type.equals(Schema.Type.ARRAY)) {
if (writeOldListStructure) {
return ConversionPatterns.listType(repetition, fieldName,
convertField("array", schema.getElementType(), REPEATED));
} else {
return ConversionPatterns.listOfElements(repetition, fieldName,
convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
}
} else if (type.equals(Schema.Type.MAP)) {
Type valType = convertField("value", schema.getValueType());
// avro map key type is always string
return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
} else if (type.equals(Schema.Type.FIXED)) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(schema.getFixedSize());
} else if (type.equals(Schema.Type.UNION)) {
return convertUnion(fieldName, schema, repetition);
} else {
throw new UnsupportedOperationException("Cannot convert Avro type " + type);
}
// schema translation can only be done for known logical types because this
// creates an equivalence
if (logicalType != null &&
!(logicalType instanceof LogicalTypes.TimestampMillis || logicalType instanceof LogicalTypes.TimestampMicros)) {
if (logicalType instanceof LogicalTypes.Decimal) {
builder = builder.as(DECIMAL)
.precision(((LogicalTypes.Decimal) logicalType).getPrecision())
.scale(((LogicalTypes.Decimal) logicalType).getScale());
} else {
OriginalType annotation = convertLogicalType(logicalType);
if (annotation != null) {
builder.as(annotation);
}
}
}
return builder.named(fieldName);
}
Example 17
| Source File: AvroSchemaConverter.java From flink with Apache License 2.0 | 4 votes |
|
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
switch (schema.getType()) {
case RECORD:
final List<Schema.Field> fields = schema.getFields();
final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
final String[] names = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
final Schema.Field field = fields.get(i);
types[i] = convertToTypeInfo(field.schema());
names[i] = field.name();
}
return Types.ROW_NAMED(names, types);
case ENUM:
return Types.STRING;
case ARRAY:
// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
case MAP:
return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
case UNION:
final Schema actualSchema;
if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(1);
} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
actualSchema = schema.getTypes().get(0);
} else if (schema.getTypes().size() == 1) {
actualSchema = schema.getTypes().get(0);
} else {
// use Kryo for serialization
return Types.GENERIC(Object.class);
}
return convertToTypeInfo(actualSchema);
case FIXED:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
// convert fixed size binary data to primitive byte arrays
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case STRING:
// convert Avro's Utf8/CharSequence to String
return Types.STRING;
case BYTES:
// logical decimal type
if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
return Types.BIG_DEC;
}
return Types.PRIMITIVE_ARRAY(Types.BYTE);
case INT:
// logical date and time type
final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
if (logicalType == LogicalTypes.date()) {
return Types.SQL_DATE;
} else if (logicalType == LogicalTypes.timeMillis()) {
return Types.SQL_TIME;
}
return Types.INT;
case LONG:
// logical timestamp type
if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
return Types.SQL_TIMESTAMP;
}
return Types.LONG;
case FLOAT:
return Types.FLOAT;
case DOUBLE:
return Types.DOUBLE;
case BOOLEAN:
return Types.BOOLEAN;
case NULL:
return Types.VOID;
}
throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
Example 18
| Source File: AvroRowDeserializationSchema.java From flink with Apache License 2.0 | 4 votes |
|
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
Example 19
| Source File: AvroSchemaConverter.java From parquet-mr with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("deprecation")
private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
Types.PrimitiveBuilder<PrimitiveType> builder;
Schema.Type type = schema.getType();
LogicalType logicalType = schema.getLogicalType();
if (type.equals(Schema.Type.BOOLEAN)) {
builder = Types.primitive(BOOLEAN, repetition);
} else if (type.equals(Schema.Type.INT)) {
builder = Types.primitive(INT32, repetition);
} else if (type.equals(Schema.Type.LONG)) {
builder = Types.primitive(INT64, repetition);
} else if (type.equals(Schema.Type.FLOAT)) {
builder = Types.primitive(FLOAT, repetition);
} else if (type.equals(Schema.Type.DOUBLE)) {
builder = Types.primitive(DOUBLE, repetition);
} else if (type.equals(Schema.Type.BYTES)) {
builder = Types.primitive(BINARY, repetition);
} else if (type.equals(Schema.Type.STRING)) {
if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
} else {
builder = Types.primitive(BINARY, repetition).as(stringType());
}
} else if (type.equals(Schema.Type.RECORD)) {
return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
} else if (type.equals(Schema.Type.ENUM)) {
builder = Types.primitive(BINARY, repetition).as(enumType());
} else if (type.equals(Schema.Type.ARRAY)) {
if (writeOldListStructure) {
return ConversionPatterns.listType(repetition, fieldName,
convertField("array", schema.getElementType(), REPEATED));
} else {
return ConversionPatterns.listOfElements(repetition, fieldName,
convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
}
} else if (type.equals(Schema.Type.MAP)) {
Type valType = convertField("value", schema.getValueType());
// avro map key type is always string
return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
} else if (type.equals(Schema.Type.FIXED)) {
builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
.length(schema.getFixedSize());
} else if (type.equals(Schema.Type.UNION)) {
return convertUnion(fieldName, schema, repetition);
} else {
throw new UnsupportedOperationException("Cannot convert Avro type " + type);
}
// schema translation can only be done for known logical types because this
// creates an equivalence
if (logicalType != null) {
if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
builder = builder.as(decimalType(decimal.getScale(), decimal.getPrecision()));
} else {
LogicalTypeAnnotation annotation = convertLogicalType(logicalType);
if (annotation != null) {
builder.as(annotation);
}
}
}
return builder.named(fieldName);
}
Example 20
| Source File: AvroRowDeserializationSchema.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private BigDecimal convertToDecimal(Schema schema, byte[] bytes) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) schema.getLogicalType();
return new BigDecimal(new BigInteger(bytes), decimalType.getScale());
}
