Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#MAP

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#MAP . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveCoercionPolicy.java    From presto with Apache License 2.0 5 votes vote down vote up
private boolean canCoerceForMap(HiveType fromHiveType, HiveType toHiveType)
{
    if (fromHiveType.getCategory() != Category.MAP || toHiveType.getCategory() != Category.MAP) {
        return false;
    }
    HiveType fromKeyType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName());
    HiveType fromValueType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName());
    HiveType toKeyType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName());
    HiveType toValueType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName());
    return (fromKeyType.equals(toKeyType) || canCoerce(fromKeyType, toKeyType)) &&
            (fromValueType.equals(toValueType) || canCoerce(fromValueType, toValueType));
}
 
Example 2
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private boolean isSupportedType(Category category) {
  return (category == Category.PRIMITIVE ||
    category == Category.LIST ||
    category == Category.STRUCT ||
    category == Category.MAP ||
    category == Category.UNION);
}
 
Example 3
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private boolean isSupportedType(Category category) {
  return (category == Category.PRIMITIVE ||
    category == Category.LIST ||
    category == Category.STRUCT ||
    category == Category.MAP ||
    category == Category.UNION);
}
 
Example 4
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Nonnull
public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi)
        throws UDFArgumentException {
    if (oi.getCategory() != Category.MAP) {
        throw new UDFArgumentException("Expected Map OI but was: " + oi);
    }
    return (MapObjectInspector) oi;
}
 
Example 5
Source File: MergeMapsUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public MergeMapsEvaluator getEvaluator(TypeInfo[] types) throws SemanticException {
    if (types.length != 1) {
        throw new UDFArgumentTypeException(types.length - 1,
            "One argument is expected but got " + types.length);
    }
    TypeInfo paramType = types[0];
    if (paramType.getCategory() != Category.MAP) {
        throw new UDFArgumentTypeException(0, "Only maps supported for now ");
    }
    return new MergeMapsEvaluator();
}
 
Example 6
Source File: SingleLevelArrayMapKeyValuesSchemaConverter.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
            return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
    if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
    }
    if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
 
Example 7
Source File: SingleLevelArraySchemaConverter.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo,
        Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
            }
        }
        else if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            if (repetition == Repetition.OPTIONAL) {
                return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
            else {
                return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
            }
        }
        else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        else {
            throw new IllegalArgumentException("Unknown type: " + typeInfo);
        }
    }
    else if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
    }
    else if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    else {
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
}
 
Example 8
Source File: MapKeyValuesSchemaConverter.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
    if (typeInfo.getCategory() == Category.PRIMITIVE) {
        if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
                    .named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
                typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
            throw new UnsupportedOperationException("Void type not implemented");
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.CHAR_TYPE_NAME)) {
            return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
                serdeConstants.VARCHAR_TYPE_NAME)) {
            return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
        }
        else if (typeInfo instanceof DecimalTypeInfo) {
            DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
            int prec = decimalTypeInfo.precision();
            int scale = decimalTypeInfo.scale();
            int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
            return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
            return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        }
        else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
            throw new UnsupportedOperationException("Unknown type not implemented");
        }
        else {
            throw new IllegalArgumentException("Unknown type: " + typeInfo);
        }
    }
    else if (typeInfo.getCategory() == Category.LIST) {
        return convertArrayType(name, (ListTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.STRUCT) {
        return convertStructType(name, (StructTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.MAP) {
        return convertMapType(name, (MapTypeInfo) typeInfo);
    }
    else if (typeInfo.getCategory() == Category.UNION) {
        throw new UnsupportedOperationException("Union type not implemented");
    }
    else {
        throw new IllegalArgumentException("Unknown type: " + typeInfo);
    }
}
 
Example 9
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
private static boolean searchAllFields(final ObjectInspector rootOI,
                                       final String name,
                                       final int[] childCounts,
                                       SearchResult position
                                       ) {
  Category category = rootOI.getCategory();
  if (category == Category.STRUCT) {
    position.index++; // first child is immediately next to parent
    StructObjectInspector sOi = (StructObjectInspector) rootOI;
    for (StructField sf : sOi.getAllStructFieldRefs()) {
      // We depend on the fact that caller takes care of calling current method
      // once for each segment in the selected column path. So, we should always get
      // searched field as immediate child
      if (position.index >= childCounts.length) {
        // input schema has more columns than what reader can read
        return false;
      }
      if (sf.getFieldName().equalsIgnoreCase(name)) {
        position.oI = sf.getFieldObjectInspector();
        return true;
      } else {
        position.index += childCounts[position.index];
      }
    }
  } else if (category == Category.MAP) {
    position.index++; // first child is immediately next to parent
    if (position.index >= childCounts.length) {
      // input schema has more columns than what reader can read
      return false;
    }
    if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
      ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
      position.oI = kOi;
      return true;
    }
    position.index += childCounts[position.index];
    if (position.index >= childCounts.length) {
      // input schema has more columns than what reader can read
      return false;
    }
    if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
      ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
      position.oI = vOi;
      return true;
    }
  }
  return false;
}
 
Example 10
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
private static boolean searchAllFields(final ObjectInspector rootOI,
                                       final String name,
                                       final int[] childCounts,
                                       SearchResult position
) {
  Category category = rootOI.getCategory();
  if (category == Category.STRUCT) {
    position.index++; // first child is immediately next to parent
    StructObjectInspector sOi = (StructObjectInspector) rootOI;
    for (StructField sf : sOi.getAllStructFieldRefs()) {
      // We depend on the fact that caller takes care of calling current method
      // once for each segment in the selected column path. So, we should always get
      // searched field as immediate child
      if (sf.getFieldName().equalsIgnoreCase(name)) {
        position.oI = sf.getFieldObjectInspector();
        return true;
      } else {
        if (position.index >= childCounts.length) {
          return false;
        }
        position.index += childCounts[position.index];
      }
    }
  } else if (category == Category.MAP) {
    position.index++; // first child is immediately next to parent
    if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
      ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
      position.oI = kOi;
      return true;
    }
    if (position.index >= childCounts.length) {
      return false;
    }
    position.index += childCounts[position.index];
    if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
      ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
      position.oI = vOi;
      return true;
    }
  }
  return false;
}
 
Example 11
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
public static boolean isMapOI(@Nonnull final ObjectInspector oi) {
    return oi.getCategory() == Category.MAP;
}
 
Example 12
Source File: CassandraColumnSerDe.java    From Hive-Cassandra with Apache License 2.0 4 votes vote down vote up
/**
 * Initialize the cassandra serialization and deserialization parameters from table properties and configuration.
 *
 * @param job
 * @param tbl
 * @param serdeName
 * @throws SerDeException
 */
@Override
protected void initCassandraSerDeParameters(Configuration job, Properties tbl, String serdeName)
    throws SerDeException {
  cassandraColumnFamily = getCassandraColumnFamily(tbl);
  cassandraColumnNames = parseOrCreateColumnMapping(tbl);

  cassandraColumnNamesBytes = new ArrayList<BytesWritable>();
  for (String columnName : cassandraColumnNames) {
    cassandraColumnNamesBytes.add(new BytesWritable(columnName.getBytes()));
  }

  iKey = cassandraColumnNames.indexOf(AbstractColumnSerDe.CASSANDRA_KEY_COLUMN);

  serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, serdeName);

  validatorType = parseOrCreateValidatorType(tbl);

  setTableMapping();

  if (cassandraColumnNames.size() != serdeParams.getColumnNames().size()) {
    throw new SerDeException(serdeName + ": columns has " +
        serdeParams.getColumnNames().size() +
        " elements while cassandra.columns.mapping has " +
        cassandraColumnNames.size() + " elements" +
        " (counting the key if implicit)");
  }

  // we just can make sure that "StandardColumn:" is mapped to MAP<String,?>
  for (int i = 0; i < cassandraColumnNames.size(); i++) {
    String cassandraColName = cassandraColumnNames.get(i);
    if (cassandraColName.endsWith(":")) {
      TypeInfo typeInfo = serdeParams.getColumnTypes().get(i);
      if ((typeInfo.getCategory() != Category.MAP) ||
          (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getTypeName()
              != Constants.STRING_TYPE_NAME)) {

        throw new SerDeException(
            serdeName + ": Cassandra column family '"
                + cassandraColName
                + "' should be mapped to map<string,?> but is mapped to "
                + typeInfo.getTypeName());
      }
    }
  }
}