org.apache.avro.Conversion Java Examples

The following examples show how to use org.apache.avro.Conversion. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ParquetAvro.java From iceberg with Apache License 2.0

7 votes

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}

Example #2

Source File: ParquetAvro.java From iceberg with Apache License 2.0

6 votes

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}

Example #3

Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testWriteDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  List<GenericRecord> expected = Arrays.asList(r1bytes, r2bytes);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as bytes",
      expected, read(GENERIC, bytesRecord, test));
}

Example #4

Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testReadDecimalBytes() throws IOException {
  Schema bytesSchema = Schema.create(Schema.Type.BYTES);
  Schema bytesRecord = record("R", field("dec", bytesSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(Schema.create(Schema.Type.BYTES));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D1, bytesSchema, DECIMAL_9_2));
  GenericRecord r2bytes = instance(bytesRecord, "dec",
      conversion.toBytes(D2, bytesSchema, DECIMAL_9_2));

  File test = write(bytesRecord, r1bytes, r2bytes);
  Assert.assertEquals("Should convert bytes to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}

Example #5

Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testWriteDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));
  List<GenericRecord> expected = Arrays.asList(r1fixed, r2fixed);

  File test = write(GENERIC, decimalRecord, r1, r2);
  Assert.assertEquals("Should read BigDecimals as fixed",
      expected, read(GENERIC, fixedRecord, test));
}

Example #6

Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testReadDecimalFixed() throws IOException {
  Schema fixedSchema = Schema.createFixed("aFixed", null, null, 4);
  Schema fixedRecord = record("R", field("dec", fixedSchema));
  Schema decimalSchema = DECIMAL_9_2.addToSchema(
      Schema.createFixed("aFixed", null, null, 4));
  Schema decimalRecord = record("R", field("dec", decimalSchema));

  GenericRecord r1 = instance(decimalRecord, "dec", D1);
  GenericRecord r2 = instance(decimalRecord, "dec", D2);
  List<GenericRecord> expected = Arrays.asList(r1, r2);

  Conversion<BigDecimal> conversion = new Conversions.DecimalConversion();

  // use the conversion directly instead of relying on the write side
  GenericRecord r1fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D1, fixedSchema, DECIMAL_9_2));
  GenericRecord r2fixed = instance(fixedRecord, "dec",
      conversion.toFixed(D2, fixedSchema, DECIMAL_9_2));

  File test = write(fixedRecord, r1fixed, r2fixed);
  Assert.assertEquals("Should convert fixed to BigDecimals",
      expected, read(GENERIC, decimalRecord, test));
}

Example #7

Source File: AvroWriteSupport.java From parquet-mr with Apache License 2.0

6 votes

private <D> Object convert(Schema schema, LogicalType logicalType,
                           Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
  }
  return datum;
}

Example #8

Source File: AvroWriteSupport.java From parquet-mr with Apache License 2.0

6 votes

@Override
public void write(T record) {
  if (rootLogicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        record.getClass(), rootLogicalType);

    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema,
        convert(rootAvroSchema, rootLogicalType, conversion, record));
    recordConsumer.endMessage();

  } else {
    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema, record);
    recordConsumer.endMessage();
  }
}

Example #9

Source File: AvroWriteSupportInt96Avro18.java From datacollector with Apache License 2.0

6 votes

private <D> Object convert(Schema schema, LogicalType logicalType,
    Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
    default: break;
  }
  return datum;
}

Example #10

Source File: AvroWriteSupportInt96Avro18.java From datacollector with Apache License 2.0

6 votes

@Override
public void write(T record) {
  if (rootLogicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        record.getClass(), rootLogicalType);

    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema,
        convert(rootAvroSchema, rootLogicalType, conversion, record));
    recordConsumer.endMessage();

  } else {
    recordConsumer.startMessage();
    writeRecordFields(rootSchema, rootAvroSchema, record);
    recordConsumer.endMessage();
  }
}

Example #11

Source File: ParquetAvro.java From iceberg with Apache License 2.0

6 votes

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}

Example #12

Source File: ParquetAvro.java From iceberg with Apache License 2.0

6 votes

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}

Example #13

Source File: AvroWriteSupport.java From parquet-mr with Apache License 2.0

5 votes

/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}

Example #14

Source File: ParentValueContainer.java From parquet-mr with Apache License 2.0

5 votes

public LogicalTypePrimitiveContainer(ParentValueContainer wrapped,
                                     Schema schema, Conversion conversion) {
  this.wrapped = wrapped;
  this.schema = schema;
  this.logicalType = schema.getLogicalType();
  this.conversion = conversion;
}

Example #15

Source File: AvroRecordConverter.java From parquet-mr with Apache License 2.0

5 votes

public AvroRecordConverter(MessageType parquetSchema, Schema avroSchema,
                           GenericData baseModel) {
  this(null, parquetSchema, avroSchema, baseModel);
  LogicalType logicalType = avroSchema.getLogicalType();
  Conversion<?> conversion = baseModel.getConversionFor(logicalType);
  this.rootContainer = ParentValueContainer.getConversionContainer(new ParentValueContainer() {
    @Override
    @SuppressWarnings("unchecked")
    public void add(Object value) {
      AvroRecordConverter.this.currentRecord = (T) value;
    }
  }, conversion, avroSchema);
}

Example #16

Source File: AvroWriteSupportInt96Avro18.java From datacollector with Apache License 2.0

5 votes

/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}

Example #17

Source File: DefaultAvroExtension.java From gradle-avro-plugin with Apache License 2.0

4 votes

@Override
public AvroExtension customConversion(Class<? extends Conversion<?>> conversionClass) {
    customConversions.add(conversionClass);
    return this;
}

Example #18

Source File: DefaultAvroExtension.java From gradle-avro-plugin with Apache License 2.0

4 votes

public void setCustomConversions(Iterable<Class<? extends Conversion<?>>> customConversions) {
    this.customConversions.set(customConversions);
}

Example #19

Source File: DefaultAvroExtension.java From gradle-avro-plugin with Apache License 2.0

4 votes

public void setCustomConversions(Provider<Iterable<Class<? extends Conversion<?>>>> provider) {
    this.customConversions.set(provider);
}

Example #20

Source File: DefaultAvroExtension.java From gradle-avro-plugin with Apache License 2.0

4 votes

@Override
public ListProperty<Class<? extends Conversion<?>>> getCustomConversions() {
    return customConversions;
}

Example #21

Source File: GenerateAvroJavaTask.java From gradle-avro-plugin with Apache License 2.0

4 votes

public void setCustomConversions(Iterable<Class<? extends Conversion<?>>> customConversions) {
    this.customConversions.set(customConversions);
}

Example #22

Source File: GenerateAvroJavaTask.java From gradle-avro-plugin with Apache License 2.0

4 votes

public void setCustomConversions(Provider<Iterable<Class<? extends Conversion<?>>>> provider) {
    this.customConversions.set(provider);
}

Example #23

Source File: GenerateAvroJavaTask.java From gradle-avro-plugin with Apache License 2.0

4 votes

@Optional
@Input
public ListProperty<Class<? extends Conversion<?>>> getCustomConversions() {
    return customConversions;
}

Example #24

Source File: AvroRecordConverter.java From parquet-mr with Apache License 2.0

4 votes

private static Converter newConverter(Schema schema, Type type,
    GenericData model, Class<?> knownClass, ParentValueContainer setter) {
  LogicalType logicalType = schema.getLogicalType();
  Conversion<?> conversion;

  if (knownClass != null) {
    conversion = model.getConversionByClass(knownClass, logicalType);
  } else {
    conversion = model.getConversionFor(logicalType);
  }

  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case INT:
    Class<?> intDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (intDatumClass == null) {
      return new AvroConverters.FieldIntegerConverter(parent);
    }
    if (intDatumClass == byte.class || intDatumClass == Byte.class) {
      return new AvroConverters.FieldByteConverter(parent);
    }
    if (intDatumClass == char.class || intDatumClass == Character.class) {
      return new AvroConverters.FieldCharConverter(parent);
    }
    if (intDatumClass == short.class || intDatumClass == Short.class) {
      return new AvroConverters.FieldShortConverter(parent);
    }
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case BYTES:
    Class<?> byteDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (byteDatumClass == null) {
      return new AvroConverters.FieldByteBufferConverter(parent);
    }
    if (byteDatumClass.isArray() && byteDatumClass.getComponentType() == byte.class) {
      return new AvroConverters.FieldByteArrayConverter(parent);
    }
    return new AvroConverters.FieldByteBufferConverter(parent);
  case STRING:
    if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName())) {
      return new AvroConverters.FieldUUIDConverter(parent, type.asPrimitiveType());
    }
    return newStringConverter(schema, model, parent);
  case RECORD:
    return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
  case ENUM:
    return new AvroConverters.FieldEnumConverter(parent, schema, model);
  case ARRAY:
    Class<?> arrayDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (arrayDatumClass != null && arrayDatumClass.isArray()) {
      return new AvroArrayConverter(parent, type.asGroupType(), schema, model,
          arrayDatumClass);
    }
    return new AvroCollectionConverter(parent, type.asGroupType(), schema,
        model, arrayDatumClass);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case FIXED:
    return new AvroConverters.FieldFixedConverter(parent, schema, model);
  default:
    throw new UnsupportedOperationException(String.format(
        "Cannot convert Avro type: %s to Parquet type: %s", schema, type));
  }
}

Example #25

Source File: AvroIndexedRecordConverter.java From parquet-mr with Apache License 2.0

4 votes

private static Converter newConverter(Schema schema, Type type,
    GenericData model, ParentValueContainer setter) {

  LogicalType logicalType = schema.getLogicalType();
  // the expected type is always null because it is determined by the parent
  // datum class, which never helps for generic. when logical types are added
  // to specific, this should pass the expected type here.
  Conversion<?> conversion = model.getConversionFor(logicalType);
  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case ARRAY:
    return new AvroArrayConverter(parent, type.asGroupType(), schema, model);
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case BYTES:
    return new AvroConverters.FieldByteBufferConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case ENUM:
    return new FieldEnumConverter(parent, schema, model);
  case FIXED:
    return new FieldFixedConverter(parent, schema, model);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case INT:
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case RECORD:
    return new AvroIndexedRecordConverter(parent, type.asGroupType(), schema, model);
  case STRING:
    return new AvroConverters.FieldStringConverter(parent);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case NULL: // fall through
  default:
    throw new UnsupportedOperationException(String.format("Cannot convert Avro type: %s" +
        " (Parquet type: %s) ", schema, type));
  }
}

Example #26

Source File: AvroExtension.java From gradle-avro-plugin with Apache License 2.0

votes

ListProperty<Class<? extends Conversion<?>>> getCustomConversions();

Example #27

Source File: AvroExtension.java From gradle-avro-plugin with Apache License 2.0

votes

AvroExtension customConversion(Class<? extends Conversion<?>> conversionClass);