org.apache.parquet.schema.DecimalMetadata Java Examples

The following examples show how to use org.apache.parquet.schema.DecimalMetadata. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ParquetRecordWriter.java From Bats with Apache License 2.0

6 votes

protected PrimitiveType getPrimitiveType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  String name = field.getName();
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (Types.isDecimalType(minorType)) {
    primitiveTypeName = logicalTypeForDecimals;
    if (usePrimitiveTypesForDecimals) {
      if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT32)) {
        primitiveTypeName = PrimitiveTypeName.INT32;
      } else if (field.getPrecision() <= ParquetTypeHelper.getMaxPrecisionForPrimitiveType(PrimitiveTypeName.INT64)) {
        primitiveTypeName = PrimitiveTypeName.INT64;
      }
    }

    length = DecimalUtility.getMaxBytesSizeForPrecision(field.getPrecision());
  }

  Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(field);
  return new PrimitiveType(repetition, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}

Example #2

Source File: ParquetResolverTest.java From pxf with Apache License 2.0

6 votes

private MessageType getParquetSchemaForPrimitiveTypes(Type.Repetition repetition, boolean readCase) {
    List<Type> fields = new ArrayList<>();

    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "s1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "s2", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "n1", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.DOUBLE, "d1", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, 16, "dc1", OriginalType.DECIMAL, new DecimalMetadata(38, 18), null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tm", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.FLOAT, "f", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT64, "bg", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BOOLEAN, "b", null));

    // GPDB only has int16 and not int8 type, so for write tiny numbers int8 are still treated as shorts in16
    OriginalType tinyType = readCase ? OriginalType.INT_8 : OriginalType.INT_16;
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "tn", tinyType));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT32, "sml", OriginalType.INT_16));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "vc1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "c1", OriginalType.UTF8));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.BINARY, "bin", null));

    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tmtz", null));
    fields.add(new PrimitiveType(repetition, PrimitiveTypeName.INT96, "tmtz2", null));

    return new MessageType("hive_schema", fields);
}

Example #3

Source File: AvroSchemaConverter190Int96Avro18.java From datacollector with Apache License 2.0

6 votes

private LogicalType convertOriginalTypeToLogicalType(OriginalType annotation, DecimalMetadata meta) {
  if (annotation == null) {
    return null;
  }
  switch (annotation) {
    case DECIMAL:
      return LogicalTypes.decimal(meta.getPrecision(), meta.getScale());
    case DATE:
      return LogicalTypes.date();
    case TIME_MILLIS:
      return LogicalTypes.timeMillis();
    case TIME_MICROS:
      return LogicalTypes.timeMicros();
    case TIMESTAMP_MILLIS:
      return LogicalTypes.timestampMillis();
    case TIMESTAMP_MICROS:
      return LogicalTypes.timestampMicros();
    default:
      return null;
  }
}

Example #4

Source File: AvroSchemaConverter190Int96Avro17.java From datacollector with Apache License 2.0

6 votes

private Map<String, String> convertOriginalTypeToMap(OriginalType annotation, DecimalMetadata meta) {
    if (annotation == null) {
      return null;
    }
    switch (annotation) {
      case DECIMAL:
        return ImmutableMap.of(
            AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL,
            AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, Integer.toString(meta.getPrecision()),
            AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, Integer.toString(meta.getScale())
        );
      case DATE:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      case TIME_MILLIS:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MILLIS);
//      case TIME_MICROS:
//        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIME_MICROS);
      case TIMESTAMP_MILLIS:
        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
//      case TIMESTAMP_MICROS:
//        return ImmutableMap.of(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MICROS);
      default:
        return null;
    }
  }

Example #5

Source File: AvroSchemaConverterLogicalTypesPre19.java From datacollector with Apache License 2.0

6 votes

private Map<String, String> convertOriginalType(OriginalType annotation, DecimalMetadata meta) {
    if (annotation == null) {
      return null;
    }
    switch (annotation) {
      case DECIMAL:
        return ImmutableMap.of(
            LOGICAL_TYPE, LOGICAL_TYPE_DECIMAL,
            LOGICAL_PROP_PRECISION, Integer.toString(meta.getPrecision()),
            LOGICAL_PROP_SCALE, Integer.toString(meta.getScale())
        );
      case DATE:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_DATE);
      case TIME_MILLIS:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIME_MILLIS);
//      case TIME_MICROS:
//        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIME_MICROS);
      case TIMESTAMP_MILLIS:
        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIMESTAMP_MILLIS);
//      case TIMESTAMP_MICROS:
//        return ImmutableMap.of(LOGICAL_TYPE, LOGICAL_TYPE_TIMESTAMP_MICROS);
      default:
        return null;
    }
  }

Example #6

Source File: ParquetTypeHelper.java From Bats with Apache License 2.0

5 votes

public static DecimalMetadata getDecimalMetadataForField(MaterializedField field) {
  switch(field.getType().getMinorType()) {
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38SPARSE:
    case DECIMAL38DENSE:
    case VARDECIMAL:
      return new DecimalMetadata(field.getPrecision(), field.getScale());
    default:
      return null;
  }
}

Example #7

Source File: ParquetTypeUtils.java From presto with Apache License 2.0

5 votes

public static Optional<DecimalType> createDecimalType(RichColumnDescriptor descriptor)
{
    if (descriptor.getPrimitiveType().getOriginalType() != DECIMAL) {
        return Optional.empty();
    }
    DecimalMetadata decimalMetadata = descriptor.getPrimitiveType().getDecimalMetadata();
    return Optional.of(DecimalType.createDecimalType(decimalMetadata.getPrecision(), decimalMetadata.getScale()));
}

Example #8

Source File: ParquetTypeHelper.java From dremio-oss with Apache License 2.0

5 votes

public static DecimalMetadata getDecimalMetadataForField(MajorType type) {
  switch (type.getMinorType()) {
    case DECIMAL:
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38SPARSE:
    case DECIMAL38DENSE:
      return new DecimalMetadata(type.getPrecision(), type.getScale());
    default:
      return null;
  }
}

Example #9

Source File: ParquetRecordWriter.java From dremio-oss with Apache License 2.0

5 votes

private PrimitiveType getPrimitiveType(Field field) {
  MajorType majorType = getMajorTypeForField(field);
  MinorType minorType = majorType.getMinorType();
  String name = field.getName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (primitiveTypeName == null) {
    return null;
  }
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  DecimalMetadata decimalMetadata  = ParquetTypeHelper.getDecimalMetadataForField(majorType);
  return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}