org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: presto   Author: prestosql   File: OrcFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
private static ObjectInspector getJavaObjectInspector(TypeInfo typeInfo)
{
    Category category = typeInfo.getCategory();
    if (category == PRIMITIVE) {
        return getPrimitiveJavaObjectInspector(getPrimitiveTypeInfo(typeInfo.getTypeName()));
    }
    if (category == LIST) {
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        return getStandardListObjectInspector(getJavaObjectInspector(listTypeInfo.getListElementTypeInfo()));
    }
    if (category == MAP) {
        MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
        return getStandardMapObjectInspector(
                getJavaObjectInspector(mapTypeInfo.getMapKeyTypeInfo()),
                getJavaObjectInspector(mapTypeInfo.getMapValueTypeInfo()));
    }
    throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled storage type: " + category);
}
 
Example #2
Source Project: presto   Author: prestosql   File: HiveBucketing.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean containsTimestampBucketedV2(TypeInfo type)
{
    switch (type.getCategory()) {
        case PRIMITIVE:
            return ((PrimitiveTypeInfo) type).getPrimitiveCategory() == TIMESTAMP;
        case LIST:
            return containsTimestampBucketedV2(((ListTypeInfo) type).getListElementTypeInfo());
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) type;
            // Note: we do not check map value type because HiveBucketingV2#hashOfMap hashes map values with v1
            return containsTimestampBucketedV2(mapTypeInfo.getMapKeyTypeInfo());
        default:
            // TODO: support more types, e.g. ROW
            throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive category: " + type.getCategory());
    }
}
 
Example #3
Source Project: presto   Author: prestosql   File: HiveWriteUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean isWritableType(TypeInfo typeInfo)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
            return isWritablePrimitiveType(primitiveCategory);
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo());
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isWritableType(listTypeInfo.getListElementTypeInfo());
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType);
    }
    return false;
}
 
Example #4
Source Project: emodb   Author: bazaarvoice   File: EmoSerDe.java    License: Apache License 2.0 6 votes vote down vote up
private Object deserializeList(ListTypeInfo type, Object data)
        throws SerDeException {
    if (!(data instanceof List)) {
        throw new SerDeException("Value not of type list");
    }
    //noinspection unchecked
    List<Object> list = (List<Object>) data;

    List<Object> values = Lists.newArrayListWithCapacity(list.size());
    for (Object entry : list) {
        Object value = deserialize(type.getListElementTypeInfo(), entry);
        values.add(value);
    }

    return values;
}
 
Example #5
Source Project: incubator-hivemall   Author: apache   File: FMPredictGenericUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Evaluator getEvaluator(TypeInfo[] typeInfo) throws SemanticException {
    if (typeInfo.length != 3) {
        throw new UDFArgumentLengthException(
            "Expected argument length is 3 but given argument length was " + typeInfo.length);
    }
    if (!HiveUtils.isNumberTypeInfo(typeInfo[0])) {
        throw new UDFArgumentTypeException(0,
            "Number type is expected for the first argument Wj: " + typeInfo[0].getTypeName());
    }
    if (typeInfo[1].getCategory() != Category.LIST) {
        throw new UDFArgumentTypeException(1,
            "List type is expected for the second argument Vjf: " + typeInfo[1].getTypeName());
    }
    ListTypeInfo typeInfo1 = (ListTypeInfo) typeInfo[1];
    if (!HiveUtils.isNumberTypeInfo(typeInfo1.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "Number type is expected for the element type of list Vjf: "
                    + typeInfo1.getTypeName());
    }
    if (!HiveUtils.isNumberTypeInfo(typeInfo[2])) {
        throw new UDFArgumentTypeException(2,
            "Number type is expected for the third argument Xj: " + typeInfo[2].getTypeName());
    }
    return new Evaluator();
}
 
Example #6
Source Project: incubator-hivemall   Author: apache   File: HitRateUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new HitRateUDAF.Evaluator();
}
 
Example #7
Source Project: incubator-hivemall   Author: apache   File: MRRUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #8
Source Project: incubator-hivemall   Author: apache   File: AUCUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    if (HiveUtils.isNumberTypeInfo(typeInfo[0]) && HiveUtils.isIntegerTypeInfo(typeInfo[1])) {
        return new ClassificationEvaluator();
    } else {
        ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
        if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
            throw new UDFArgumentTypeException(0,
                "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
        }

        ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
        if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
            throw new UDFArgumentTypeException(1,
                "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
        }

        return new RankingEvaluator();
    }
}
 
Example #9
Source Project: incubator-hivemall   Author: apache   File: PrecisionUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #10
Source Project: incubator-hivemall   Author: apache   File: MAPUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #11
Source Project: incubator-hivemall   Author: apache   File: RecallUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #12
Source Project: incubator-hivemall   Author: apache   File: NDCGUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GenericUDAFEvaluator getEvaluator(@Nonnull TypeInfo[] typeInfo)
        throws SemanticException {
    if (typeInfo.length != 2 && typeInfo.length != 3) {
        throw new UDFArgumentTypeException(typeInfo.length - 1,
            "_FUNC_ takes two or three arguments");
    }

    ListTypeInfo arg1type = HiveUtils.asListTypeInfo(typeInfo[0]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg1type.getListElementTypeInfo())
            && !HiveUtils.isStructTypeInfo(arg1type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(0,
            "The first argument `array rankItems` is invalid form: " + typeInfo[0]);
    }
    ListTypeInfo arg2type = HiveUtils.asListTypeInfo(typeInfo[1]);
    if (!HiveUtils.isPrimitiveTypeInfo(arg2type.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "The second argument `array correctItems` is invalid form: " + typeInfo[1]);
    }

    return new Evaluator();
}
 
Example #13
Source Project: Cobol-to-Hive   Author: rbheemana   File: CobolDeserializer.java    License: Apache License 2.0 6 votes vote down vote up
private Object worker(String columnName, TypeInfo columnType){
	
	switch(columnType.getCategory()) {
		
		case STRUCT:
			return deserializeStruct(columnName, (StructTypeInfo) columnType);
		case UNION:
			return deserializeUnion(columnName,(UnionTypeInfo) columnType);
		case LIST:
		return deserializeList(columnName, (ListTypeInfo) columnType);
		case MAP:
		throw new RuntimeException("map type is not possible for cobol layout" + columnType.getCategory());
		case PRIMITIVE:
		return deserializePrimitive(columnName, (PrimitiveTypeInfo) columnType);
		default:
		throw new RuntimeException("Unknown TypeInfo: " + columnType.getCategory());
	}
}
 
Example #14
Source Project: Cobol-to-Hive   Author: rbheemana   File: CobolDeserializer.java    License: Apache License 2.0 6 votes vote down vote up
private Object deserializeList(String columnName, ListTypeInfo columnType)
		throws RuntimeException {
	int size = Integer.parseInt(rowElements.get(propertiesList.get(fieldNo).get("dcol")));
	
	List<Object> listContents = new ArrayList<Object>();
	TypeInfo ti = columnType.getListElementTypeInfo();
	String tn = columnType.getTypeName();
	rowElements.add("");
	int tempfieldNo = fieldNo, fieldNoList=fieldNo; 
	for (int j = 0; j < size; j++) {
			listContents.add(worker(tn,ti));
			fieldNoList = fieldNo;
			fieldNo = tempfieldNo;
	}
	fieldNo = fieldNoList;
	return listContents;

}
 
Example #15
Source Project: bigdata-tutorial   Author: micmiu   File: JSONCDHSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses a JSON object according to the Hive column's type.
 *
 * @param field         - The JSON object to parse
 * @param fieldTypeInfo - Metadata about the Hive column
 * @return - The parsed value of the field
 */
private Object parseField(Object field, TypeInfo fieldTypeInfo) {
	switch (fieldTypeInfo.getCategory()) {
		case PRIMITIVE:
			// Jackson will return the right thing in this case, so just return
			// the object
			if (field instanceof String) {
				field = field.toString().replaceAll("\n", "\\\\n");
			}
			return field;
		case LIST:
			return parseList(field, (ListTypeInfo) fieldTypeInfo);
		case MAP:
			return parseMap(field, (MapTypeInfo) fieldTypeInfo);
		case STRUCT:
			return parseStruct(field, (StructTypeInfo) fieldTypeInfo);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example #16
Source Project: incubator-retired-blur   Author: apache   File: BlurObjectInspectorGenerator.java    License: Apache License 2.0 6 votes vote down vote up
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
  switch (ti.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
  case STRUCT:
    StructTypeInfo sti = (StructTypeInfo) ti;
    List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
    for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
      ois.add(createObjectInspectorWorker(typeInfo));
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
  case LIST:
    ListTypeInfo lti = (ListTypeInfo) ti;
    TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
    return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
  default:
    throw new SerDeException("No Hive categories matched for [" + ti + "]");
  }
}
 
Example #17
Source Project: presto   Author: prestosql   File: HiveCoercionRecordCursor.java    License: Apache License 2.0 5 votes vote down vote up
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor)
{
    requireNonNull(typeManager, "typeManage is null");
    requireNonNull(fromHiveType, "fromHiveType is null");
    requireNonNull(toHiveType, "toHiveType is null");
    this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null");
    HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    this.fromElementType = fromElementHiveType.getType(typeManager);
    this.toType = toHiveType.getType(typeManager);
    this.toElementType = toElementHiveType.getType(typeManager);
    this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType, bridgingRecordCursor);
    this.pageBuilder = elementCoercer == null ? null : new PageBuilder(ImmutableList.of(toType));
}
 
Example #18
Source Project: presto   Author: prestosql   File: HiveType.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean isSupportedType(TypeInfo typeInfo, StorageFormat storageFormat)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null;
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isSupportedType(mapTypeInfo.getMapKeyTypeInfo(), storageFormat) && isSupportedType(mapTypeInfo.getMapValueTypeInfo(), storageFormat);
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isSupportedType(listTypeInfo.getListElementTypeInfo(), storageFormat);
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream()
                    .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
        case UNION:
            // This feature (reading uniontypes as structs) has only been verified against Avro and ORC tables. Here's a discussion:
            //   1. Avro tables are supported and verified.
            //   2. ORC tables are supported and verified.
            //   3. The Parquet format doesn't support uniontypes itself so there's no need to add support for it in Presto.
            //   4. TODO: RCFile tables are not supported yet.
            //   5. TODO: The support for Avro is done in SerDeUtils so it's possible that formats other than Avro are also supported. But verification is needed.
            if (storageFormat.getSerDe().equalsIgnoreCase(AVRO.getSerDe()) || storageFormat.getSerDe().equalsIgnoreCase(ORC.getSerDe())) {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                return unionTypeInfo.getAllUnionObjectTypeInfos().stream()
                        .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
            }
    }
    return false;
}
 
Example #19
Source Project: presto   Author: prestosql   File: HivePageSource.java    License: Apache License 2.0 5 votes vote down vote up
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType)
{
    requireNonNull(typeManager, "typeManage is null");
    requireNonNull(fromHiveType, "fromHiveType is null");
    requireNonNull(toHiveType, "toHiveType is null");
    HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType);
}
 
Example #20
Source Project: presto   Author: prestosql   File: HiveBucketingV2.java    License: Apache License 2.0 5 votes vote down vote up
private static int hashOfList(ListTypeInfo type, Block singleListBlock)
{
    TypeInfo elementTypeInfo = type.getListElementTypeInfo();
    int result = 0;
    for (int i = 0; i < singleListBlock.getPositionCount(); i++) {
        result = result * 31 + hash(elementTypeInfo, singleListBlock, i);
    }
    return result;
}
 
Example #21
Source Project: presto   Author: prestosql   File: HiveBucketingV1.java    License: Apache License 2.0 5 votes vote down vote up
private static int hashOfList(ListTypeInfo type, Block singleListBlock)
{
    TypeInfo elementTypeInfo = type.getListElementTypeInfo();
    int result = 0;
    for (int i = 0; i < singleListBlock.getPositionCount(); i++) {
        result = result * 31 + hash(elementTypeInfo, singleListBlock, i);
    }
    return result;
}
 
Example #22
Source Project: presto   Author: prestosql   File: HiveCoercionPolicy.java    License: Apache License 2.0 5 votes vote down vote up
private boolean canCoerceForList(HiveType fromHiveType, HiveType toHiveType)
{
    if (fromHiveType.getCategory() != Category.LIST || toHiveType.getCategory() != Category.LIST) {
        return false;
    }
    HiveType fromElementType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    HiveType toElementType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName());
    return fromElementType.equals(toElementType) || canCoerce(fromElementType, toElementType);
}
 
Example #23
Source Project: flink   Author: flink-tpc-ds   File: HiveTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Convert Hive data type to a Flink data type.
 *
 * @param hiveType a Hive data type
 * @return the corresponding Flink data type
 */
public static DataType toFlinkType(TypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getCategory()) {
		case PRIMITIVE:
			return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
		case LIST:
			ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
			return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
			return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;

			List<String> names = structTypeInfo.getAllStructFieldNames();
			List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();

			DataTypes.Field[] fields = new DataTypes.Field[names.size()];

			for (int i = 0; i < fields.length; i++) {
				fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
			}

			return DataTypes.ROW(fields);
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive data type %s yet.", hiveType));
	}
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: HiveInspectors.java    License: Apache License 2.0 5 votes vote down vote up
private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}
 
Example #25
Source Project: multiple-dimension-spread   Author: yahoojapan   File: MDSListObjectInspector.java    License: Apache License 2.0 5 votes vote down vote up
public MDSListObjectInspector( final ListTypeInfo typeInfo ){
  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getListElementTypeInfo() ); 
  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getListElementTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example #26
Source Project: dremio-oss   Author: dremio   File: HiveTypeCoercion.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeCoercion getChildTypeCoercion(String fieldName, BatchSchema childSchema) {
  Map<String, TypeInfo> childrenTypeInfoMap = CaseInsensitiveMap.newHashMap();

  TypeInfo typeInfo = typeInfoMap.get(fieldName);
  if (typeInfo instanceof StructTypeInfo) {
    for (Field field: childSchema.getFields()) {
      childrenTypeInfoMap.put(field.getName(), ((StructTypeInfo) typeInfo).getStructFieldTypeInfo(field.getName()));
    }
  } else if (typeInfo instanceof ListTypeInfo) {
    childrenTypeInfoMap.put(childSchema.getFields().get(0).getName(), ((ListTypeInfo) typeInfo).getListElementTypeInfo());
  }
  return new HiveTypeCoercion(childrenTypeInfoMap, varcharTruncationEnabled);
}
 
Example #27
Source Project: emodb   Author: bazaarvoice   File: EmoSerDe.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Deserializes a raw value to the provided type.
 */
private Object deserialize(TypeInfo type, Object rawValue)
        throws SerDeException {
    Object value = null;

    if (rawValue != null) {
        switch (type.getCategory()) {
            case PRIMITIVE:
                value = deserializePrimitive((PrimitiveTypeInfo) type, rawValue);
                break;
            case STRUCT:
                value = deserializeStruct((StructTypeInfo) type, rawValue);
                break;
            case MAP:
                value = deserializeMap((MapTypeInfo) type, rawValue);
                break;
            case LIST:
                value = deserializeList((ListTypeInfo) type, rawValue);
                break;
            case UNION:
                value = deserializeUnion((UnionTypeInfo) type, rawValue);
                break;
        }
    }

    return value;
}
 
Example #28
Source Project: emr-dynamodb-connector   Author: awslabs   File: HiveDynamoDBListType.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean supportsHiveType(TypeInfo typeInfo) {
  if (typeInfo.getCategory() != ObjectInspector.Category.LIST) {
    return false;
  }

  TypeInfo elementTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo();
  try {
    HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(elementTypeInfo);
  } catch (IllegalArgumentException e) {
    return false;
  }
  return true;
}
 
Example #29
Source Project: incubator-hivemall   Author: apache   File: FFMPredictGenericUDAF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Evaluator getEvaluator(@Nonnull TypeInfo[] typeInfo) throws SemanticException {
    if (typeInfo.length != 5) {
        throw new UDFArgumentLengthException(
            "Expected argument length is 5 but given argument length was " + typeInfo.length);
    }
    if (!HiveUtils.isNumberTypeInfo(typeInfo[0])) {
        throw new UDFArgumentTypeException(0,
            "Number type is expected for the first argument Wi: " + typeInfo[0].getTypeName());
    }
    if (typeInfo[1].getCategory() != Category.LIST) {
        throw new UDFArgumentTypeException(1,
            "List type is expected for the second argument Vifj: " + typeInfo[1].getTypeName());
    }
    if (typeInfo[2].getCategory() != Category.LIST) {
        throw new UDFArgumentTypeException(2,
            "List type is expected for the third argument Vjfi: " + typeInfo[2].getTypeName());
    }
    ListTypeInfo typeInfo1 = (ListTypeInfo) typeInfo[1];
    if (!HiveUtils.isFloatingPointTypeInfo(typeInfo1.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(1,
            "Double or Float type is expected for the element type of list Vifj: "
                    + typeInfo1.getTypeName());
    }
    ListTypeInfo typeInfo2 = (ListTypeInfo) typeInfo[2];
    if (!HiveUtils.isFloatingPointTypeInfo(typeInfo2.getListElementTypeInfo())) {
        throw new UDFArgumentTypeException(2,
            "Double or Float type is expected for the element type of list Vjfi: "
                    + typeInfo1.getTypeName());
    }
    if (!HiveUtils.isNumberTypeInfo(typeInfo[3])) {
        throw new UDFArgumentTypeException(3,
            "Number type is expected for the third argument Xi: " + typeInfo[3].getTypeName());
    }
    if (!HiveUtils.isNumberTypeInfo(typeInfo[4])) {
        throw new UDFArgumentTypeException(4,
            "Number type is expected for the third argument Xi: " + typeInfo[4].getTypeName());
    }
    return new Evaluator();
}
 
Example #30
Source Project: incubator-hivemall   Author: apache   File: HiveUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean isFloatingPointListTypeInfo(@Nonnull TypeInfo typeInfo) {
    if (typeInfo.getCategory() != Category.LIST) {
        return false;
    }
    TypeInfo elemTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo();
    return isFloatingPointTypeInfo(elemTypeInfo);
}