org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: presto   Author: prestosql   File: S3SelectPushdown.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean areColumnTypesSupported(List<Column> columns)
{
    requireNonNull(columns, "columns is null");

    if (columns.isEmpty()) {
        return false;
    }

    for (Column column : columns) {
        String type = column.getType().getHiveTypeName().toString();
        if (column.getType().getTypeInfo() instanceof DecimalTypeInfo) {
            // skip precision and scale when check decimal type
            type = DECIMAL_TYPE_NAME;
        }
        if (!SUPPORTED_COLUMN_TYPES.contains(type)) {
            return false;
        }
    }

    return true;
}
 
Example #2
Source Project: presto   Author: prestosql   File: TestDataWritableWriter.java    License: Apache License 2.0 6 votes vote down vote up
private Binary decimalToBinary(HiveDecimal hiveDecimal, DecimalTypeInfo decimalTypeInfo)
{
    int prec = decimalTypeInfo.precision();
    int scale = decimalTypeInfo.scale();
    byte[] decimalBytes = hiveDecimal.setScale(scale).unscaledValue().toByteArray();

    // Estimated number of bytes needed.
    int precToBytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
    if (precToBytes == decimalBytes.length) {
        // No padding needed.
        return Binary.fromByteArray(decimalBytes);
    }

    byte[] tgt = new byte[precToBytes];
    if (hiveDecimal.signum() == -1) {
        // For negative number, initializing bits to 1
        for (int i = 0; i < precToBytes; i++) {
            tgt[i] |= 0xFF;
        }
    }

    System.arraycopy(decimalBytes, 0, tgt, precToBytes - decimalBytes.length, decimalBytes.length); // Padding leading zeroes/ones.
    return Binary.fromByteArray(tgt);
}
 
Example #3
Source Project: presto   Author: prestosql   File: TestParquetDecimalScaling.java    License: Apache License 2.0 6 votes vote down vote up
public ObjectInspector getParquetObjectInspector()
{
    if (!forceFixedLengthArray && precision > 0 && precision < 10) {
        return javaIntObjectInspector;
    }

    if (!forceFixedLengthArray && precision >= 10 && precision < 18) {
        return javaLongObjectInspector;
    }

    if (precision > 38 || precision < 0) {
        throw new IllegalArgumentException("Scale cannot be greater than 38 or less than 0");
    }

    return new JavaHiveDecimalObjectInspector(new DecimalTypeInfo(precision, scale));
}
 
Example #4
Source Project: metacat   Author: Netflix   File: HiveTypeConverter.java    License: Apache License 2.0 6 votes vote down vote up
private static Type getPrimitiveType(final ObjectInspector fieldInspector) {
    final PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) fieldInspector)
        .getPrimitiveCategory();
    if (HiveTypeMapping.getHIVE_TO_CANONICAL().containsKey(primitiveCategory.name())) {
        return HiveTypeMapping.getHIVE_TO_CANONICAL().get(primitiveCategory.name());
    }
    switch (primitiveCategory) {
        case DECIMAL:
            final DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) ((PrimitiveObjectInspector) fieldInspector)
                .getTypeInfo();
            return DecimalType.createDecimalType(decimalTypeInfo.precision(), decimalTypeInfo.getScale());
        case CHAR:
            final int cLength = ((CharTypeInfo) ((PrimitiveObjectInspector)
                fieldInspector).getTypeInfo()).getLength();
            return CharType.createCharType(cLength);
        case VARCHAR:
            final int vLength = ((VarcharTypeInfo) ((PrimitiveObjectInspector) fieldInspector)
                .getTypeInfo()).getLength();
            return VarcharType.createVarcharType(vLength);
        default:
            return null;
    }
}
 
Example #5
Source Project: presto   Author: prestosql   File: HiveType.java    License: Apache License 2.0 5 votes vote down vote up
public static Type getPrimitiveType(PrimitiveTypeInfo typeInfo)
{
    switch (typeInfo.getPrimitiveCategory()) {
        case BOOLEAN:
            return BOOLEAN;
        case BYTE:
            return TINYINT;
        case SHORT:
            return SMALLINT;
        case INT:
            return INTEGER;
        case LONG:
            return BIGINT;
        case FLOAT:
            return REAL;
        case DOUBLE:
            return DOUBLE;
        case STRING:
            return createUnboundedVarcharType();
        case VARCHAR:
            return createVarcharType(((VarcharTypeInfo) typeInfo).getLength());
        case CHAR:
            return createCharType(((CharTypeInfo) typeInfo).getLength());
        case DATE:
            return DATE;
        case TIMESTAMP:
            return TIMESTAMP;
        case BINARY:
            return VARBINARY;
        case DECIMAL:
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            return createDecimalType(decimalTypeInfo.precision(), decimalTypeInfo.scale());
        default:
            return null;
    }
}
 
Example #6
Source Project: presto   Author: prestosql   File: AbstractTestParquetReader.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testParquetLongDecimalWriteToPrestoDecimalWithNonMatchingScale()
        throws Exception
{
    tester.testRoundTrip(
            new JavaHiveDecimalObjectInspector(new DecimalTypeInfo(38, 10)),
            ImmutableList.of(HiveDecimal.create(100 * longTenToNth(10), 10)),
            ImmutableList.of(new SqlDecimal(BigInteger.valueOf(100 * longTenToNth(9)), 38, 9)),
            createDecimalType(38, 9));
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: HiveTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static DataType toFlinkPrimitiveType(PrimitiveTypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getPrimitiveCategory()) {
		case CHAR:
			return DataTypes.CHAR(((CharTypeInfo) hiveType).getLength());
		case VARCHAR:
			return DataTypes.VARCHAR(((VarcharTypeInfo) hiveType).getLength());
		case STRING:
			return DataTypes.STRING();
		case BOOLEAN:
			return DataTypes.BOOLEAN();
		case BYTE:
			return DataTypes.TINYINT();
		case SHORT:
			return DataTypes.SMALLINT();
		case INT:
			return DataTypes.INT();
		case LONG:
			return DataTypes.BIGINT();
		case FLOAT:
			return DataTypes.FLOAT();
		case DOUBLE:
			return DataTypes.DOUBLE();
		case DATE:
			return DataTypes.DATE();
		case TIMESTAMP:
			return DataTypes.TIMESTAMP();
		case BINARY:
			return DataTypes.BYTES();
		case DECIMAL:
			DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveType;
			return DataTypes.DECIMAL(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale());
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive primitive type %s yet", hiveType));
	}
}
 
Example #8
Source Project: dremio-oss   Author: dremio   File: ManagedHiveSchema.java    License: Apache License 2.0 5 votes vote down vote up
public ManagedHiveSchema(final JobConf jobConf, final HiveReaderProto.HiveTableXattr tableXattr) {
  final java.util.Properties tableProperties = new java.util.Properties();
  HiveUtilities.addProperties(jobConf, tableProperties, HiveReaderProtoUtil.getTableProperties(tableXattr));
  final String fieldNameProp = Optional.ofNullable(tableProperties.getProperty("columns")).orElse("");
  final String fieldTypeProp = Optional.ofNullable(tableProperties.getProperty("columns.types")).orElse("");
  varcharTruncationEnabled = HiveDatasetOptions
      .enforceVarcharWidth(HiveReaderProtoUtil.convertValuesToNonProtoAttributeValues(tableXattr.getDatasetOptionMap()));

  final Iterator<String> fieldNames = Splitter.on(",").trimResults().split(fieldNameProp).iterator();
  final Iterator<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString(fieldTypeProp).iterator();

  final Map<String, ManagedSchemaField> schemaFieldMap = new HashMap<>();
  final Map<String, TypeInfo> typeInfoMap = new HashMap<>();
  while (fieldNames.hasNext() && fieldTypes.hasNext()) {
    final String fieldName = fieldNames.next();
    final TypeInfo fieldType = fieldTypes.next();
    ManagedSchemaField field;
    if (fieldType instanceof DecimalTypeInfo) {
      field = ManagedSchemaField.newFixedLenField(fieldName, fieldType.getTypeName(),
        ((DecimalTypeInfo) fieldType).getPrecision(), ((DecimalTypeInfo) fieldType).getScale());
      typeInfoMap.put(fieldName, fieldType);
    } else if (fieldType instanceof BaseCharTypeInfo) {
      if (varcharTruncationEnabled) {
        field = ManagedSchemaField.newFixedLenField(fieldName, fieldType.getTypeName(),
            ((BaseCharTypeInfo) fieldType).getLength(), 0);
        typeInfoMap.put(fieldName, fieldType);
      } else {
        field = ManagedSchemaField.newUnboundedLenField(fieldName, fieldType.getTypeName());
      }
    } else {
      // Extend ManagedSchemaField.java in case granular information has to be stored.
      // No mention of len and scale means it is unbounded. So, we store max values.
      field = ManagedSchemaField.newUnboundedLenField(fieldName, fieldType.getTypeName());
      typeInfoMap.put(fieldName, fieldType);
    }
    schemaFieldMap.put(fieldName, field);
  }
  fieldInfo = CaseInsensitiveMap.newImmutableMap(schemaFieldMap);
  typeInfo = CaseInsensitiveMap.newImmutableMap(typeInfoMap);
}
 
Example #9
Source Project: dremio-oss   Author: dremio   File: HiveORCVectorizedReader.java    License: Apache License 2.0 5 votes vote down vote up
private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 
Example #10
Source Project: dremio-oss   Author: dremio   File: HiveORCVectorizedReader.java    License: Apache License 2.0 5 votes vote down vote up
private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
    case BOOLEAN:
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case DATE:
      return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case TIMESTAMP:
      return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case FLOAT:
    case DOUBLE:
      return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    case DECIMAL:
      DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
      return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
        tInfo.precision(), tInfo.scale()
      );
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
        .build(logger);
    }
}
 
Example #11
Source Project: flink   Author: apache   File: HiveTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static DataType toFlinkPrimitiveType(PrimitiveTypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getPrimitiveCategory()) {
		case CHAR:
			return DataTypes.CHAR(((CharTypeInfo) hiveType).getLength());
		case VARCHAR:
			return DataTypes.VARCHAR(((VarcharTypeInfo) hiveType).getLength());
		case STRING:
			return DataTypes.STRING();
		case BOOLEAN:
			return DataTypes.BOOLEAN();
		case BYTE:
			return DataTypes.TINYINT();
		case SHORT:
			return DataTypes.SMALLINT();
		case INT:
			return DataTypes.INT();
		case LONG:
			return DataTypes.BIGINT();
		case FLOAT:
			return DataTypes.FLOAT();
		case DOUBLE:
			return DataTypes.DOUBLE();
		case DATE:
			return DataTypes.DATE();
		case TIMESTAMP:
			return DataTypes.TIMESTAMP(9);
		case BINARY:
			return DataTypes.BYTES();
		case DECIMAL:
			DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) hiveType;
			return DataTypes.DECIMAL(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale());
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive primitive type %s yet", hiveType));
	}
}
 
Example #12
Source Project: nifi   Author: apache   File: TestNiFiOrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test_getOrcField_decimal() {
    // given
    final DecimalTypeInfo expected = TypeInfoFactory.getDecimalTypeInfo(4, 2);
    final DataType decimalDataType = RecordFieldType.DECIMAL.getDecimalDataType(4, 2);

    // when
    final TypeInfo orcField = NiFiOrcUtils.getOrcField(decimalDataType, false);

    // then
    Assert.assertEquals(expected, orcField);
}
 
Example #13
Source Project: presto   Author: prestosql   File: OrcTester.java    License: Apache License 2.0 4 votes vote down vote up
private static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return javaStringObjectInspector;
    }
    if (type instanceof CharType) {
        int charLength = ((CharType) type).getLength();
        return new JavaHiveCharObjectInspector(getCharTypeInfo(charLength));
    }
    if (type instanceof VarbinaryType) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (type instanceof ArrayType) {
        return getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (type instanceof MapType) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (type instanceof RowType) {
        return getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toList()),
                type.getTypeParameters().stream()
                        .map(OrcTester::getJavaObjectInspector)
                        .collect(toList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example #14
Source Project: presto   Author: prestosql   File: TypeConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static TypeInfo toHiveTypeInfo(Type type)
{
    if (BOOLEAN.equals(type)) {
        return HIVE_BOOLEAN.getTypeInfo();
    }
    if (BIGINT.equals(type)) {
        return HIVE_LONG.getTypeInfo();
    }
    if (INTEGER.equals(type)) {
        return HIVE_INT.getTypeInfo();
    }
    if (SMALLINT.equals(type)) {
        return HIVE_SHORT.getTypeInfo();
    }
    if (TINYINT.equals(type)) {
        return HIVE_BYTE.getTypeInfo();
    }
    if (REAL.equals(type)) {
        return HIVE_FLOAT.getTypeInfo();
    }
    if (DOUBLE.equals(type)) {
        return HIVE_DOUBLE.getTypeInfo();
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        if (varcharType.isUnbounded()) {
            return HIVE_STRING.getTypeInfo();
        }
        if (varcharType.getBoundedLength() <= HiveVarchar.MAX_VARCHAR_LENGTH) {
            return getVarcharTypeInfo(varcharType.getBoundedLength());
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported VARCHAR types: VARCHAR(<=%d), VARCHAR.", type, HiveVarchar.MAX_VARCHAR_LENGTH));
    }
    if (type instanceof CharType) {
        CharType charType = (CharType) type;
        int charLength = charType.getLength();
        if (charLength <= HiveChar.MAX_CHAR_LENGTH) {
            return getCharTypeInfo(charLength);
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported CHAR types: CHAR(<=%d).",
                type, HiveChar.MAX_CHAR_LENGTH));
    }
    if (VARBINARY.equals(type)) {
        return HIVE_BINARY.getTypeInfo();
    }
    if (DATE.equals(type)) {
        return HIVE_DATE.getTypeInfo();
    }
    if (TIMESTAMP.equals(type)) {
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
        // Hive does not have TIMESTAMP_WITH_TIME_ZONE, this is just a work around for iceberg.
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
    }
    if (isArrayType(type)) {
        TypeInfo elementType = toHiveTypeInfo(type.getTypeParameters().get(0));
        return getListTypeInfo(elementType);
    }
    if (isMapType(type)) {
        TypeInfo keyType = toHiveTypeInfo(type.getTypeParameters().get(0));
        TypeInfo valueType = toHiveTypeInfo(type.getTypeParameters().get(1));
        return getMapTypeInfo(keyType, valueType);
    }
    if (isRowType(type)) {
        ImmutableList.Builder<String> fieldNames = ImmutableList.builder();
        for (TypeSignatureParameter parameter : type.getTypeSignature().getParameters()) {
            if (!parameter.isNamedTypeSignature()) {
                throw new IllegalArgumentException(format("Expected all parameters to be named type, but got %s", parameter));
            }
            NamedTypeSignature namedTypeSignature = parameter.getNamedTypeSignature();
            if (namedTypeSignature.getName().isEmpty()) {
                throw new PrestoException(NOT_SUPPORTED, format("Anonymous row type is not supported in Hive. Please give each field a name: %s", type));
            }
            fieldNames.add(namedTypeSignature.getName().get());
        }
        return getStructTypeInfo(
                fieldNames.build(),
                type.getTypeParameters().stream()
                        .map(TypeConverter::toHiveTypeInfo)
                        .collect(toList()));
    }
    throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s", type));
}
 
Example #15
Source Project: presto   Author: prestosql   File: RcFileTester.java    License: Apache License 2.0 4 votes vote down vote up
private static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return javaStringObjectInspector;
    }
    if (type.equals(VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (type instanceof ArrayType) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (type instanceof MapType) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (type instanceof RowType) {
        return getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toList()),
                type.getTypeParameters().stream()
                        .map(RcFileTester::getJavaObjectInspector)
                        .collect(toList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example #16
Source Project: presto   Author: prestosql   File: HiveTypeTranslator.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeInfo translate(Type type)
{
    if (BOOLEAN.equals(type)) {
        return HIVE_BOOLEAN.getTypeInfo();
    }
    if (BIGINT.equals(type)) {
        return HIVE_LONG.getTypeInfo();
    }
    if (INTEGER.equals(type)) {
        return HIVE_INT.getTypeInfo();
    }
    if (SMALLINT.equals(type)) {
        return HIVE_SHORT.getTypeInfo();
    }
    if (TINYINT.equals(type)) {
        return HIVE_BYTE.getTypeInfo();
    }
    if (REAL.equals(type)) {
        return HIVE_FLOAT.getTypeInfo();
    }
    if (DOUBLE.equals(type)) {
        return HIVE_DOUBLE.getTypeInfo();
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        if (varcharType.isUnbounded()) {
            return HIVE_STRING.getTypeInfo();
        }
        if (varcharType.getBoundedLength() <= HiveVarchar.MAX_VARCHAR_LENGTH) {
            return getVarcharTypeInfo(varcharType.getBoundedLength());
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported VARCHAR types: VARCHAR(<=%d), VARCHAR.", type, HiveVarchar.MAX_VARCHAR_LENGTH));
    }
    if (type instanceof CharType) {
        CharType charType = (CharType) type;
        int charLength = charType.getLength();
        if (charLength <= HiveChar.MAX_CHAR_LENGTH) {
            return getCharTypeInfo(charLength);
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported CHAR types: CHAR(<=%d).",
                type, HiveChar.MAX_CHAR_LENGTH));
    }
    if (VARBINARY.equals(type)) {
        return HIVE_BINARY.getTypeInfo();
    }
    if (DATE.equals(type)) {
        return HIVE_DATE.getTypeInfo();
    }
    if (TIMESTAMP.equals(type)) {
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
    }
    if (isArrayType(type)) {
        TypeInfo elementType = translate(type.getTypeParameters().get(0));
        return getListTypeInfo(elementType);
    }
    if (isMapType(type)) {
        TypeInfo keyType = translate(type.getTypeParameters().get(0));
        TypeInfo valueType = translate(type.getTypeParameters().get(1));
        return getMapTypeInfo(keyType, valueType);
    }
    if (isRowType(type)) {
        ImmutableList.Builder<String> fieldNames = ImmutableList.builder();
        for (TypeSignatureParameter parameter : type.getTypeSignature().getParameters()) {
            if (!parameter.isNamedTypeSignature()) {
                throw new IllegalArgumentException(format("Expected all parameters to be named type, but got %s", parameter));
            }
            NamedTypeSignature namedTypeSignature = parameter.getNamedTypeSignature();
            if (namedTypeSignature.getName().isEmpty()) {
                throw new PrestoException(NOT_SUPPORTED, format("Anonymous row type is not supported in Hive. Please give each field a name: %s", type));
            }
            fieldNames.add(namedTypeSignature.getName().get());
        }
        return getStructTypeInfo(
                fieldNames.build(),
                type.getTypeParameters().stream()
                        .map(this::translate)
                        .collect(toImmutableList()));
    }
    throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s", type));
}
 
Example #17
Source Project: presto   Author: prestosql   File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BooleanType.BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BigintType.BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(IntegerType.INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TinyintType.TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(RealType.REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return writableStringObjectInspector;
    }
    if (type instanceof CharType) {
        return writableHiveCharObjectInspector;
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DateType.DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (isArrayType(type)) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (isMapType(type)) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (isRowType(type)) {
        return ObjectInspectorFactory.getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toImmutableList()),
                type.getTypeParameters().stream()
                        .map(HiveWriteUtils::getJavaObjectInspector)
                        .collect(toImmutableList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example #18
Source Project: presto   Author: prestosql   File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getRowColumnInspector(Type type)
{
    if (type.equals(BooleanType.BOOLEAN)) {
        return writableBooleanObjectInspector;
    }

    if (type.equals(BigintType.BIGINT)) {
        return writableLongObjectInspector;
    }

    if (type.equals(IntegerType.INTEGER)) {
        return writableIntObjectInspector;
    }

    if (type.equals(SmallintType.SMALLINT)) {
        return writableShortObjectInspector;
    }

    if (type.equals(TinyintType.TINYINT)) {
        return writableByteObjectInspector;
    }

    if (type.equals(RealType.REAL)) {
        return writableFloatObjectInspector;
    }

    if (type.equals(DoubleType.DOUBLE)) {
        return writableDoubleObjectInspector;
    }

    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        if (varcharType.isUnbounded()) {
            // Unbounded VARCHAR is not supported by Hive.
            // Values for such columns must be stored as STRING in Hive
            return writableStringObjectInspector;
        }
        if (varcharType.getBoundedLength() <= HiveVarchar.MAX_VARCHAR_LENGTH) {
            // VARCHAR columns with the length less than or equal to 65535 are supported natively by Hive
            return getPrimitiveWritableObjectInspector(getVarcharTypeInfo(varcharType.getBoundedLength()));
        }
    }

    if (isCharType(type)) {
        CharType charType = (CharType) type;
        int charLength = charType.getLength();
        return getPrimitiveWritableObjectInspector(getCharTypeInfo(charLength));
    }

    if (type.equals(VarbinaryType.VARBINARY)) {
        return writableBinaryObjectInspector;
    }

    if (type.equals(DateType.DATE)) {
        return writableDateObjectInspector;
    }

    if (type.equals(TimestampType.TIMESTAMP)) {
        return writableTimestampObjectInspector;
    }

    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveWritableObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }

    if (isArrayType(type) || isMapType(type) || isRowType(type)) {
        return getJavaObjectInspector(type);
    }

    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example #19
Source Project: presto   Author: prestosql   File: SingleLevelArrayMapKeyValuesSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
            return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
    if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
 
Example #20
Source Project: presto   Author: prestosql   File: TestDataWritableWriter.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * It writes the primitive value to the Parquet RecordConsumer.
 *
 * @param value The object that contains the primitive value.
 * @param inspector The object inspector used to get the correct value type.
 */
private void writePrimitive(Object value, PrimitiveObjectInspector inspector)
{
    if (value == null) {
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case VOID:
            return;
        case DOUBLE:
            recordConsumer.addDouble(((DoubleObjectInspector) inspector).get(value));
            break;
        case BOOLEAN:
            recordConsumer.addBoolean(((BooleanObjectInspector) inspector).get(value));
            break;
        case FLOAT:
            recordConsumer.addFloat(((FloatObjectInspector) inspector).get(value));
            break;
        case BYTE:
            recordConsumer.addInteger(((ByteObjectInspector) inspector).get(value));
            break;
        case INT:
            recordConsumer.addInteger(((IntObjectInspector) inspector).get(value));
            break;
        case LONG:
            recordConsumer.addLong(((LongObjectInspector) inspector).get(value));
            break;
        case SHORT:
            recordConsumer.addInteger(((ShortObjectInspector) inspector).get(value));
            break;
        case STRING:
            String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromString(v));
            break;
        case CHAR:
            String vChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(value).getStrippedValue();
            recordConsumer.addBinary(Binary.fromString(vChar));
            break;
        case VARCHAR:
            String vVarchar = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(value).getValue();
            recordConsumer.addBinary(Binary.fromString(vVarchar));
            break;
        case BINARY:
            byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(Binary.fromByteArray(vBinary));
            break;
        case TIMESTAMP:
            Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary());
            break;
        case DECIMAL:
            HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value));
            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
            recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo));
            break;
        case DATE:
            Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
            recordConsumer.addInteger(DateWritable.dateToDays(vDate));
            break;
        default:
            throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());
    }
}
 
Example #21
Source Project: presto   Author: prestosql   File: SingleLevelArraySchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo,
        Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
        }
        else if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
        }
        else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        else {
            throw new IllegalArgumentException("Unknown type: " + typeInfo);
        }
    }
    else if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    else {
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
}
 
Example #22
Source Project: presto   Author: prestosql   File: MapKeyValuesSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        else if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        else {
            throw new IllegalArgumentException("Unknown type: " + typeInfo);
        }
    }
    else if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    else {
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
}
 
Example #23
Source Project: marble   Author: 51nb   File: TypeInferenceUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static RelDataType getRelDataType(ObjectInspector oi,
      RelDataTypeFactory factory) {
    if (oi == null) {
      return factory.createTypeWithNullability(
          factory.createSqlType(SqlTypeName.ANY), true);
    }
    ObjectInspector.Category category = oi.getCategory();
    if (!category.equals(PRIMITIVE)) {
      if (category.equals(ObjectInspector.Category.LIST)) {
        StandardListObjectInspector standardListObjectInspector =
            (StandardListObjectInspector) oi;
        return factory.createArrayType(
            getRelDataType(
                standardListObjectInspector.getListElementObjectInspector(),
                factory), -1);
      } else {
        throw new IllegalArgumentException(
            "unsupported ObjectInspector category :" + category);
      }

    }
    PrimitiveObjectInspector primitiveObjectInspector =
        (PrimitiveObjectInspector) oi;
    PrimitiveTypeInfo primitiveTypeInfo =
        primitiveObjectInspector.getTypeInfo();

    SqlTypeName sqlTypeName = HIVE_TYPE_2_CALCITE_SQL_TYPE_INFO.get(
        primitiveTypeInfo);
    //handle DecimalTypeInfo special case
    if (primitiveTypeInfo instanceof DecimalTypeInfo) {
      sqlTypeName = SqlTypeName.DECIMAL;
//      return factory.createTypeWithNullability(
//          factory.createSqlType(
//              sqlTypeName, ((DecimalTypeInfo) primitiveTypeInfo).getPrecision(),
//              ((DecimalTypeInfo) primitiveTypeInfo).getScale()),
//          true);
    }
    if (sqlTypeName == null) {
      throw new RuntimeException(
          "can't get sqlType for hive primitiveTypeInfo:" + primitiveTypeInfo);
    }
    return factory.createTypeWithNullability(
        factory.createSqlType(
            sqlTypeName),
        true);

  }
 
Example #24
Source Project: dremio-oss   Author: dremio   File: HiveSchemaConverter.java    License: Apache License 2.0 4 votes vote down vote up
public static Field getArrowFieldFromHivePrimitiveType(String name, TypeInfo typeInfo) {
  switch (typeInfo.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    switch (pTypeInfo.getPrimitiveCategory()) {
    case BOOLEAN:

      return new Field(name, true, new Bool(), null);
    case BYTE:
      return new Field(name, true, new Int(32, true), null);
    case SHORT:
      return new Field(name, true, new Int(32, true), null);

    case INT:
      return new Field(name, true, new Int(32, true), null);

    case LONG:
      return new Field(name, true, new Int(64, true), null);

    case FLOAT:
      return new Field(name, true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);

    case DOUBLE:
      return new Field(name, true, new FloatingPoint(FloatingPointPrecision.DOUBLE), null);

    case DATE:
      return new Field(name, true, new Date(DateUnit.MILLISECOND), null);

    case TIMESTAMP:
      return new Field(name, true, new Timestamp(TimeUnit.MILLISECOND, null), null);

    case BINARY:
      return new Field(name, true, new Binary(), null);
    case DECIMAL: {
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) pTypeInfo;
      return new Field(name, true, new Decimal(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale()), null);
    }

    case STRING:
    case VARCHAR:
    case CHAR: {
      return new Field(name, true, new Utf8(), null);
    }
    case UNKNOWN:
    case VOID:
    default:
      // fall through.
    }
  default:
  }

  return null;
}
 
Example #25
Source Project: Cobol-to-Hive   Author: rbheemana   File: CobolNumberField.java    License: Apache License 2.0 4 votes vote down vote up
@Override
	public Object deserialize(byte[] rowBytes) throws CobolSerdeException {
		byte[] temp = transcodeField(super.getBytes(rowBytes));
		String s1 = new String(temp);

		if (this.compType > 0) {
			if (this.compType == 3) {
				s1 = unpackData(super.getBytes(rowBytes), this.decimalLocation);
			}else if(this.compType == 4){
				s1 = getBinary(super.getBytes(rowBytes), this.decimalLocation);
			}
		} //} else if (this.decimalLocation > 0) {
		else {
			//Now calling unpackSign on all numeric fields for which compType resolves to 0.
			//
			//The function will check to see if the least significant byte has been overpunched with a sign and
			//return a negative number if a negative sign is found.
			s1 = unpackSign(super.getBytes(rowBytes), this.decimalLocation);
		}
//		else if (this.decimalLocation > 0) {
//			s1 = s1.substring(0, this.length * this.divideFactor
//					- this.decimalLocation)
//					+ "."
//					+ s1.substring(this.length * this.divideFactor
//							- this.decimalLocation);
//		}
//		System.out.println(name + "\t - " + s1 + "\t:" + offset + "\[email protected]"
//				+ length);
		try {
			switch (((PrimitiveTypeInfo) this.typeInfo).getPrimitiveCategory()) {
			case LONG:
				return Long.parseLong(s1.trim());
			case SHORT:
				return Short.parseShort(s1.trim());
			case INT:
				return Integer.parseInt(s1.trim());
			case BYTE:
				return Byte.parseByte(s1.trim());
			case FLOAT:
				return Float.parseFloat(s1.trim());
			case DOUBLE:
				return Double.parseDouble(s1.trim());
			case DECIMAL:
				BigDecimal bd = new BigDecimal(s1);
				HiveDecimal dec = HiveDecimal.create(bd);
				JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory
						.getPrimitiveJavaObjectInspector((DecimalTypeInfo) this.typeInfo);
				return oi.set(null, dec);
			}
		} catch (Exception e) {
			return null; // if cannot be converted make it null
		}
		return null;

	}