org.apache.parquet.schema.DecimalMetadata Java Examples

protected PrimitiveType getPrimitiveType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  String name = field.getName();
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (Types.isDecimalType(minorType)) {
    primitiveTypeName = logicalTypeForDecimals;
    if (usePrimitiveTypesForDecimals) {
      if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT32)) {
        primitiveTypeName = PrimitiveTypeName.INT32;
      } else if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT64)) {
        primitiveTypeName = PrimitiveTypeName.INT64;
      }
    }

    length = DecimalUtility.getMaxBytesSizeForPrecision(field.getPrecision());
  }

  Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(field);
  return new PrimitiveType(repetition, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
 

private MessageType getParquetSchemaForPrimitiveTypes(Type.Repetition repetition, boolean readCase) {
    List<Type> fields = new ArrayList<>();

    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "s1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "s2", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "n1", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.DOUBLE, "d1", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, 16, "dc1", OriginalType.DECIMAL, new DecimalMetadata(38, 18), null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tm", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.FLOAT, "f", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT64, "bg", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BOOLEAN, "b", null));

    // GPDB only has int16 and not int8 type, so for write tiny numbers int8 are still treated as shorts in16
    OriginalType tinyType = readCase ? OriginalType.INT_8 : OriginalType.INT_16;
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "tn", tinyType));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "sml", OriginalType.INT_16));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "vc1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "c1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "bin", null));

    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tmtz", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tmtz2", null));

    return new MessageType("hive_schema", fields);
}
 

private LogicalType convertOriginalTypeToLogicalType(OriginalType annotation, DecimalMetadata meta) {
  if (annotation == null) {
    return null;
  }
  switch (annotation) {
    case DECIMAL:
      return LogicalTypes.decimal(meta.getPrecision(), meta.getScale());
    case DATE:
      return LogicalTypes.date();
    case TIME_MILLIS:
      return LogicalTypes.timeMillis();
    case TIME_MICROS:
      return LogicalTypes.timeMicros();
    case TIMESTAMP_MILLIS:
      return LogicalTypes.timestampMillis();
    case TIMESTAMP_MICROS:
      return LogicalTypes.timestampMicros();
    default:
      return null;
  }
}
 

private Map<String, String> convertOriginalTypeToMap(OriginalType annotation, DecimalMetadata meta) {
    if (annotation == null) {
      return null;
    }
    switch (annotation) {
      case DECIMAL:
        return ImmutableMap.of(
            AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL,
            AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, Integer.toString(meta.getPrecision()),
            AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, Integer.toString(meta.getScale())
        );
      case DATE:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      case TIME_MILLIS:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MILLIS);
//      case TIME_MICROS:
//        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MICROS);
      case TIMESTAMP_MILLIS:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
//      case TIMESTAMP_MICROS:
//        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MICROS);
      default:
        return null;
    }
  }
 

private Map<String, String> convertOriginalType(OriginalType annotation, DecimalMetadata meta) {
    if (annotation == null) {
      return null;
    }
    switch (annotation) {
      case DECIMAL:
        return ImmutableMap.of(
            LOGICAL_TYPE, LOGICAL_TYPE_DECIMAL,
            LOGICAL_PROP_PRECISION, Integer.toString(meta.getPrecision()),
            LOGICAL_PROP_SCALE, Integer.toString(meta.getScale())
        );
      case DATE:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_DATE);
      case TIME_MILLIS:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIME_MILLIS);
//      case TIME_MICROS:
//        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIME_MICROS);
      case TIMESTAMP_MILLIS:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIMESTAMP_MILLIS);
//      case TIMESTAMP_MICROS:
//        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIMESTAMP_MICROS);
      default:
        return null;
    }
  }
 

public static DecimalMetadata getDecimalMetadataForField(MaterializedField field) {
  switch(field.getType().getMinorType()) {
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38SPARSE:
    case DECIMAL38DENSE:
    case VARDECIMAL:
      return new DecimalMetadata(field.getPrecision(), field.getScale());
    default:
      return null;
  }
}
 

public static Optional<DecimalType> createDecimalType(RichColumnDescriptor descriptor)
{
    if (descriptor.getPrimitiveType().getOriginalType() != DECIMAL) {
        return Optional.empty();
    }
    DecimalMetadata decimalMetadata = descriptor.getPrimitiveType().getDecimalMetadata();
    return Optional.of(DecimalType.createDecimalType(decimalMetadata.getPrecision(), decimalMetadata.getScale()));
}
 

public static DecimalMetadata getDecimalMetadataForField(MajorType type) {
  switch (type.getMinorType()) {
    case DECIMAL:
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38SPARSE:
    case DECIMAL38DENSE:
      return new DecimalMetadata(type.getPrecision(), type.getScale());
    default:
      return null;
  }
}
 

private PrimitiveType getPrimitiveType(Field field) {
  MajorType majorType = getMajorTypeForField(field);
  MinorType minorType = majorType.getMinorType();
  String name = field.getName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (primitiveTypeName == null) {
    return null;
  }
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  DecimalMetadata decimalMetadata  = ParquetTypeHelper.getDecimalMetadataForField(majorType);
  return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}