org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CobolDeserializer.java    From Cobol-to-Hive with Apache License 2.0 6 votes vote down vote up
private Object worker(String columnName, TypeInfo columnType){
	
	switch(columnType.getCategory()) {
		
		case STRUCT:
			return deserializeStruct(columnName, (StructTypeInfo) columnType);
		case UNION:
			return deserializeUnion(columnName,(UnionTypeInfo) columnType);
		case LIST:
		return deserializeList(columnName, (ListTypeInfo) columnType);
		case MAP:
		throw new RuntimeException("map type is not possible for cobol layout" + columnType.getCategory());
		case PRIMITIVE:
		return deserializePrimitive(columnName, (PrimitiveTypeInfo) columnType);
		default:
		throw new RuntimeException("Unknown TypeInfo: " + columnType.getCategory());
	}
}
 
Example #2
Source File: MDSMapObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
  TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
  if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
    keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
  }
  else{
    throw new RuntimeException( "Map key type is string only." );
  }

  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() ); 

  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example #3
Source File: UnionField.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public UnionField( final UnionTypeInfo typeInfo ){
  byte tag = (byte)0;
  
  for( TypeInfo childTypeInfo : typeInfo.getAllUnionObjectTypeInfos() ){
    IGetUnionObject field = craeteGetUnionObject( tag , childTypeInfo );
    columnTypeMap.put( field.getColumnType() , field );
    tag++;
  }
}
 
Example #4
Source File: OrcUnion.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
public OrcUnionObjectInspector(UnionTypeInfo info) {
  List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos();
  this.children = new ArrayList<ObjectInspector>(unionChildren.size());
  for(TypeInfo child: info.getAllUnionObjectTypeInfos()) {
    this.children.add(OrcLazyObjectInspectorUtils.createWritableObjectInspector(child));
  }
}
 
Example #5
Source File: OrcLazyUnionObjectInspector.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
public OrcLazyUnionObjectInspector(UnionTypeInfo info) {
  List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos();
  this.children = new ArrayList<ObjectInspector>(unionChildren.size());
  for(TypeInfo child: info.getAllUnionObjectTypeInfos()) {
    this.children.add(OrcLazyObjectInspectorUtils.createWritableObjectInspector(child));
  }
}
 
Example #6
Source File: EmoSerDe.java    From emodb with Apache License 2.0 5 votes vote down vote up
private Object deserializeUnion(UnionTypeInfo type, Object data)
        throws SerDeException {
    // Try each union type in order until one matches.
    for (byte i=0; i < type.getAllUnionObjectTypeInfos().size(); i++) {
        try {
            Object value = deserialize(type.getAllUnionObjectTypeInfos().get(i), data);
            return new StandardUnionObjectInspector.StandardUnion(i, value);
        } catch (SerDeException e) {
            // Skip it and try the next
        }
    }

    throw new SerDeException("No suitable type found");
}
 
Example #7
Source File: EmoSerDe.java    From emodb with Apache License 2.0 5 votes vote down vote up
/**
 * Deserializes a raw value to the provided type.
 */
private Object deserialize(TypeInfo type, Object rawValue)
        throws SerDeException {
    Object value = null;

    if (rawValue != null) {
        switch (type.getCategory()) {
            case PRIMITIVE:
                value = deserializePrimitive((PrimitiveTypeInfo) type, rawValue);
                break;
            case STRUCT:
                value = deserializeStruct((StructTypeInfo) type, rawValue);
                break;
            case MAP:
                value = deserializeMap((MapTypeInfo) type, rawValue);
                break;
            case LIST:
                value = deserializeList((ListTypeInfo) type, rawValue);
                break;
            case UNION:
                value = deserializeUnion((UnionTypeInfo) type, rawValue);
                break;
        }
    }

    return value;
}
 
Example #8
Source File: MDSListObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public MDSListObjectInspector( final ListTypeInfo typeInfo ){
  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getListElementTypeInfo() ); 
  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getListElementTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example #9
Source File: MDSStructObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public MDSUnionStructField( final int fieldId , final String fieldName , final StandardUnionObjectInspector inspector , final UnionTypeInfo unionTypeInfo ){
  this.fieldId = fieldId;
  this.fieldName = fieldName;
  this.inspector = inspector;
  childColumnAndIndex = new ColumnAndIndex();
  childColumnAndIndex.columnIndex = -1;
  unionField = new UnionField( unionTypeInfo );
}
 
Example #10
Source File: HiveType.java    From presto with Apache License 2.0 5 votes vote down vote up
public static boolean isSupportedType(TypeInfo typeInfo, StorageFormat storageFormat)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null;
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isSupportedType(mapTypeInfo.getMapKeyTypeInfo(), storageFormat) && isSupportedType(mapTypeInfo.getMapValueTypeInfo(), storageFormat);
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isSupportedType(listTypeInfo.getListElementTypeInfo(), storageFormat);
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream()
                    .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
        case UNION:
            // This feature (reading uniontypes as structs) has only been verified against Avro and ORC tables. Here's a discussion:
            //   1. Avro tables are supported and verified.
            //   2. ORC tables are supported and verified.
            //   3. The Parquet format doesn't support uniontypes itself so there's no need to add support for it in Presto.
            //   4. TODO: RCFile tables are not supported yet.
            //   5. TODO: The support for Avro is done in SerDeUtils so it's possible that formats other than Avro are also supported. But verification is needed.
            if (storageFormat.getSerDe().equalsIgnoreCase(AVRO.getSerDe()) || storageFormat.getSerDe().equalsIgnoreCase(ORC.getSerDe())) {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                return unionTypeInfo.getAllUnionObjectTypeInfos().stream()
                        .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
            }
    }
    return false;
}
 
Example #11
Source File: MDSMapObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public UnionGetField( final UnionTypeInfo unionTypeInfo ){
  unionField = new UnionField( unionTypeInfo );
}
 
Example #12
Source File: MDSListObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public UnionGetField( final UnionTypeInfo unionTypeInfo ){
  unionField = new UnionField( unionTypeInfo );
}
 
Example #13
Source File: MDSObjectInspectorFactory.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector craeteObjectInspectorFromTypeInfo( final TypeInfo typeInfo ){
  switch ( typeInfo.getCategory() ){
    case STRUCT:
      return new MDSStructObjectInspector( (StructTypeInfo)typeInfo );
    case MAP:
      return new MDSMapObjectInspector( (MapTypeInfo)typeInfo );
    case LIST:
      return new MDSListObjectInspector( (ListTypeInfo)typeInfo );
    case UNION:
      UnionTypeInfo unionTypeInfo = (UnionTypeInfo)typeInfo;
      List<ObjectInspector> unionList = new ArrayList<ObjectInspector>();
      for( TypeInfo childTypeInfo : unionTypeInfo.getAllUnionObjectTypeInfos() ){
        unionList.add( craeteObjectInspectorFromTypeInfo( childTypeInfo ) );
      }
      return ObjectInspectorFactory.getStandardUnionObjectInspector( unionList );
    case PRIMITIVE:
      PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo)typeInfo;
      switch( primitiveTypeInfo.getPrimitiveCategory() ){
        case STRING:
          return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
        case BINARY:
          return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
        case BYTE:
          return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
        case FLOAT:
          return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
        case SHORT:
          return PrimitiveObjectInspectorFactory.writableShortObjectInspector;

        case DATE:
        case DECIMAL:
        case TIMESTAMP:
        case VOID:
        default:
        throw new UnsupportedOperationException( "Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory() );
      }
    default:
      throw new UnsupportedOperationException( "Unknown category " + typeInfo.getCategory() );
  }
}
 
Example #14
Source File: CobolDeserializer.java    From Cobol-to-Hive with Apache License 2.0 4 votes vote down vote up
private Object deserializeUnion(String columnName,UnionTypeInfo columnType) throws RuntimeException {
	return null;
}
 
Example #15
Source File: OrcLazyObjectInspectorUtils.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector createLazyObjectInspector(TypeInfo info) {
  switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return FLOAT_OBJECT_INSPECTOR;
        case DOUBLE:
          return DOUBLE_OBJECT_INSPECTOR;
        case BOOLEAN:
          return BOOLEAN_OBJECT_INSPECTOR;
        case BYTE:
          return BYTE_OBJECT_INSPECTOR;
        case SHORT:
          return SHORT_OBJECT_INSPECTOR;
        case INT:
          return INT_OBJECT_INSPECTOR;
        case LONG:
          return LONG_OBJECT_INSPECTOR;
        case BINARY:
          return BINARY_OBJECT_INSPECTOR;
        case STRING:
          return STRING_OBJECT_INSPECTOR;
        case TIMESTAMP:
          return TIMESTAMP_OBJECT_INSPECTOR;
        default:
          throw new IllegalArgumentException("Unknown primitive type " +
            ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new OrcLazyStructObjectInspector((StructTypeInfo) info);
    case UNION:
      return new OrcLazyUnionObjectInspector((UnionTypeInfo) info);
    case MAP:
      return new OrcLazyMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new OrcLazyListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 
Example #16
Source File: OrcLazyObjectInspectorUtils.java    From hive-dwrf with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector createWritableObjectInspector(TypeInfo info) {
  switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
        case BYTE:
          return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
        case SHORT:
          return PrimitiveObjectInspectorFactory.writableShortObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
        case BINARY:
          return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
        case STRING:
          return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
        case TIMESTAMP:
          return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
        default:
          throw new IllegalArgumentException("Unknown primitive type " +
            ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new OrcStruct.OrcStructInspector((StructTypeInfo) info);
    case UNION:
      return new OrcUnion.OrcUnionObjectInspector((UnionTypeInfo) info);
    case MAP:
      return new OrcStruct.OrcMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new OrcStruct.OrcListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 
Example #17
Source File: HiveSchemaConverter.java    From kite with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
static Schema convert(LinkedList<String> path, String name,
                      TypeInfo type, Collection<String[]> required) {
  switch (type.getCategory()) {
    case PRIMITIVE:
      if (type.getClass() == charClass || type.getClass() == varcharClass) {
        // this is required because type name includes length
        return Schema.create(Schema.Type.STRING);
      }

      String typeInfoName = type.getTypeName();
      Preconditions.checkArgument(TYPEINFO_TO_TYPE.containsKey(typeInfoName),
          "Cannot convert unsupported type: %s", typeInfoName);
      return Schema.create(TYPEINFO_TO_TYPE.get(typeInfoName));

    case LIST:
      return Schema.createArray(optional(convert(path, name,
          ((ListTypeInfo) type).getListElementTypeInfo(), required)));

    case MAP:
      MapTypeInfo mapType = (MapTypeInfo) type;
      Preconditions.checkArgument(
          "string".equals(mapType.getMapKeyTypeInfo().toString()),
          "Non-String map key type: %s", mapType.getMapKeyTypeInfo());

      return Schema.createMap(optional(convert(path, name,
          mapType.getMapValueTypeInfo(), required)));

    case STRUCT:
      return convert(path, name, (StructTypeInfo) type, required);

    case UNION:
      List<TypeInfo> unionTypes = ((UnionTypeInfo) type)
          .getAllUnionObjectTypeInfos();

      // add NULL so all union types are optional
      List<Schema> types = Lists.newArrayList(NULL);
      for (int i = 0; i < unionTypes.size(); i += 1) {
        // types within unions cannot be required
        types.add(convert(
            path, name + "_" + i, unionTypes.get(i), NO_REQUIRED_FIELDS));
      }

      return Schema.createUnion(types);

    default:
      throw new IllegalArgumentException(
          "Unknown TypeInfo category: " + type.getCategory());
  }
}