Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static Block serializeObject(Type type, BlockBuilder builder, Object object, ObjectInspector inspector, boolean filterNullMapKeys)
{
    switch (inspector.getCategory()) {
        case PRIMITIVE:
            serializePrimitive(type, builder, object, (PrimitiveObjectInspector) inspector);
            return null;
        case LIST:
            return serializeList(type, builder, object, (ListObjectInspector) inspector);
        case MAP:
            return serializeMap(type, builder, object, (MapObjectInspector) inspector, filterNullMapKeys);
        case STRUCT:
            return serializeStruct(type, builder, object, (StructObjectInspector) inspector);
        case UNION:
            return serializeUnion(type, builder, object, (UnionObjectInspector) inspector);
    }
    throw new RuntimeException("Unknown object inspector category: " + inspector.getCategory());
}
 
Example 2
public MapConverter(ObjectInspector inputOI, SettableMapObjectInspector outputOI) {
  if (inputOI instanceof MapObjectInspector) {
    this.inputOI = (MapObjectInspector) inputOI;
    this.outputOI = outputOI;
    inputKeyOI = this.inputOI.getMapKeyObjectInspector();
    outputKeyOI = outputOI.getMapKeyObjectInspector();
    inputValueOI = this.inputOI.getMapValueObjectInspector();
    outputValueOI = outputOI.getMapValueObjectInspector();
    keyConverter = getConverter(inputKeyOI, outputKeyOI);
    valueConverter = getConverter(inputValueOI, outputValueOI);
  } else if (!(inputOI instanceof VoidObjectInspector)) {
    throw new UnsupportedOperationException(
        "Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName()
            + "not supported yet.");
  }
}
 
Example 3
Source Project: localization_nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable) throws IOException {
    super(columnId, inspector, writer, nullable);
    this.isDirectV2 = isNewWriteFormat(writer);
    MapObjectInspector insp = (MapObjectInspector) inspector;
    childrenWriters = new TreeWriter[2];
    childrenWriters[0] =
            createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
    childrenWriters[1] =
            createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
    lengths = createIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
    recordPosition(rowIndexPosition);
}
 
Example 4
Source Project: localization_nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        MapObjectInspector insp = (MapObjectInspector) inspector;
        // this sucks, but it will have to do until we can get a better
        // accessor in the MapObjectInspector.
        Map<?, ?> valueMap = insp.getMap(obj);
        lengths.write(valueMap.size());
        if (createBloomFilter) {
            bloomFilter.addLong(valueMap.size());
        }
        for (Map.Entry<?, ?> entry : valueMap.entrySet()) {
            childrenWriters[0].write(entry.getKey());
            childrenWriters[1].write(entry.getValue());
        }
    }
}
 
Example 5
Source Project: pxf   Source File: HiveResolver.java    License: Apache License 2.0 6 votes vote down vote up
private List<OneField> traverseMap(Object obj, MapObjectInspector moi)
        throws BadRecordException, IOException {
    List<OneField> complexRecord = new LinkedList<>();
    List<OneField> mapRecord = new LinkedList<>();
    ObjectInspector koi = moi.getMapKeyObjectInspector();
    ObjectInspector voi = moi.getMapValueObjectInspector();
    Map<?, ?> map = moi.getMap(obj);
    if (map == null) {
        throw new BadRecordException(
                "Illegal value NULL for Hive data type Map");
    } else if (map.isEmpty()) {
        traverseTuple(null, koi, complexRecord, true);
        traverseTuple(null, voi, complexRecord, true);
        addOneFieldToRecord(mapRecord, DataType.TEXT,
                HdfsUtilities.toString(complexRecord, mapkeyDelim));
    } else {
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            traverseTuple(entry.getKey(), koi, complexRecord, true);
            traverseTuple(entry.getValue(), voi, complexRecord, true);
            addOneFieldToRecord(mapRecord, DataType.TEXT,
                    HdfsUtilities.toString(complexRecord, mapkeyDelim));
            complexRecord.clear();
        }
    }
    return mapRecord;
}
 
Example 6
@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  MapObjectInspector mapObjectInspector = (MapObjectInspector) objectInspector;
  ObjectInspector mapKeyObjectInspector = mapObjectInspector.getMapKeyObjectInspector();
  ObjectInspector mapValueObjectInspector = mapObjectInspector.getMapValueObjectInspector();
  TypeInfo mapKeyTypeInfo = ((MapTypeInfo) odpsTypeInfo).getKeyTypeInfo();
  TypeInfo mapValueTypeInfo = ((MapTypeInfo) odpsTypeInfo).getValueTypeInfo();

  Map map = mapObjectInspector.getMap(o);
  Map<Object, Object> newMap = new HashMap<>();
  for (Object k : map.keySet()) {
    Object v = map.get(k);
    newMap.put(HiveObjectConverter.convert(mapKeyObjectInspector, k, mapKeyTypeInfo),
        HiveObjectConverter.convert(mapValueObjectInspector, v, mapValueTypeInfo));
  }

  return newMap;
}
 
Example 7
Source Project: dremio-oss   Source File: HiveORCVectorizedReader.java    License: Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example 8
Source Project: dremio-oss   Source File: HiveORCVectorizedReader.java    License: Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example 9
Source Project: incubator-hivemall   Source File: SlimUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}
 
Example 10
Source Project: incubator-hivemall   Source File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 11
Source Project: incubator-hivemall   Source File: MapKeyValuesUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 1) {
        throw new UDFArgumentLengthException(
            "The function MAP_KEYS only accepts one argument.");
    } else if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentTypeException(0,
            "\"" + Category.MAP.toString().toLowerCase()
                    + "\" is expected at function MAP_KEYS, " + "but \""
                    + arguments[0].getTypeName() + "\" is found");
    }

    this.mapOI = (MapObjectInspector) arguments[0];

    List<String> structFieldNames = new ArrayList<String>();
    List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
    structFieldNames.add("key");
    structFieldObjectInspectors.add(mapOI.getMapKeyObjectInspector());
    structFieldNames.add("value");
    structFieldObjectInspectors.add(mapOI.getMapValueObjectInspector());

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames,
            structFieldObjectInspectors));
}
 
Example 12
Source Project: incubator-hivemall   Source File: MapTailNUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
            "map_tail_n only takes 2 arguments: map<object, object>, int");
    }
    if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentException("The first argument must be a map");
    }
    this.mapObjectInspector = (MapObjectInspector) arguments[0];
    if (!(arguments[1] instanceof IntObjectInspector)) {
        throw new UDFArgumentException("The second argument must be an int");
    }
    this.intObjectInspector = (IntObjectInspector) arguments[1];

    ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(
        mapObjectInspector.getMapKeyObjectInspector());
    ObjectInspector valueOI = mapObjectInspector.getMapValueObjectInspector();

    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}
 
Example 13
Source Project: bigdata-tutorial   Source File: JSONCDHSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example 14
Source Project: searchanalytics-bigdata   Source File: JSONSerDe.java    License: MIT License 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 
Example 15
Source Project: nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable) throws IOException {
    super(columnId, inspector, writer, nullable);
    this.isDirectV2 = isNewWriteFormat(writer);
    MapObjectInspector insp = (MapObjectInspector) inspector;
    childrenWriters = new TreeWriter[2];
    childrenWriters[0] =
            createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
    childrenWriters[1] =
            createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
    lengths = createIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
    recordPosition(rowIndexPosition);
}
 
Example 16
Source Project: nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        MapObjectInspector insp = (MapObjectInspector) inspector;
        // this sucks, but it will have to do until we can get a better
        // accessor in the MapObjectInspector.
        Map<?, ?> valueMap = insp.getMap(obj);
        lengths.write(valueMap.size());
        if (createBloomFilter) {
            bloomFilter.addLong(valueMap.size());
        }
        for (Map.Entry<?, ?> entry : valueMap.entrySet()) {
            childrenWriters[0].write(entry.getKey());
            childrenWriters[1].write(entry.getValue());
        }
    }
}
 
Example 17
Source Project: hive-dwrf   Source File: WriterImpl.java    License: Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable, Configuration conf,
              boolean useVInts, boolean lowMemoryMode,
              MemoryEstimate memoryEstimate) throws IOException {
  super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
  MapObjectInspector insp = (MapObjectInspector) inspector;
  childrenWriters = new TreeWriter[2];
  childrenWriters[0] =
    createTreeWriter(insp.getMapKeyObjectInspector(), writer, true, conf, useVInts,
        lowMemoryMode, memoryEstimate);
  childrenWriters[1] =
    createTreeWriter(insp.getMapValueObjectInspector(), writer, true, conf, useVInts,
        lowMemoryMode, memoryEstimate);
  lengths =
    new RunLengthIntegerWriter(writer.createStream(columnId,
        OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
  recordPosition(rowIndexPosition);
}
 
Example 18
Source Project: hive-dwrf   Source File: WriterImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
  long rawDataSize = 0;
  if (obj != null) {
    MapObjectInspector insp = (MapObjectInspector) inspector;
    // this sucks, but it will have to do until we can get a better
    // accessor in the MapObjectInspector.
    Map<?, ?> valueMap = insp.getMap(obj);
    // Don't use getMapSize(), it's inconsistent for some object inspectors
    int len = valueMap.size();
    lengths.write(len);
    for(Map.Entry<?, ?> entry: valueMap.entrySet()) {
      childrenWriters[0].write(entry.getKey());
      childrenWriters[1].write(entry.getValue());
      rawDataSize += childrenWriters[0].getRowRawDataSize();
      rawDataSize += childrenWriters[1].getRowRawDataSize();
    }
  }
  super.write(obj, rawDataSize);
}
 
Example 19
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static Block serializeMap(Type type, BlockBuilder builder, Object object, MapObjectInspector inspector, boolean filterNullMapKeys)
{
    Map<?, ?> map = inspector.getMap(object);
    if (map == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter");
    Type keyType = typeParameters.get(0);
    Type valueType = typeParameters.get(1);
    ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
    ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
    BlockBuilder currentBuilder;

    boolean builderSynthesized = false;
    if (builder == null) {
        builderSynthesized = true;
        builder = type.createBlockBuilder(null, 1);
    }
    currentBuilder = builder.beginBlockEntry();

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        // Hive skips map entries with null keys
        if (!filterNullMapKeys || entry.getKey() != null) {
            serializeObject(keyType, currentBuilder, entry.getKey(), keyInspector);
            serializeObject(valueType, currentBuilder, entry.getValue(), valueInspector);
        }
    }

    builder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(builder, 0);
    }
    else {
        return null;
    }
}
 
Example 20
Source Project: presto   Source File: TestHiveFileFormats.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean hasType(ObjectInspector objectInspector, PrimitiveCategory... types)
{
    if (objectInspector instanceof PrimitiveObjectInspector) {
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) objectInspector;
        PrimitiveCategory primitiveCategory = primitiveInspector.getPrimitiveCategory();
        for (PrimitiveCategory type : types) {
            if (primitiveCategory == type) {
                return true;
            }
        }
        return false;
    }
    if (objectInspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) objectInspector;
        return hasType(listInspector.getListElementObjectInspector(), types);
    }
    if (objectInspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) objectInspector;
        return hasType(mapInspector.getMapKeyObjectInspector(), types) ||
                hasType(mapInspector.getMapValueObjectInspector(), types);
    }
    if (objectInspector instanceof StructObjectInspector) {
        for (StructField field : ((StructObjectInspector) objectInspector).getAllStructFieldRefs()) {
            if (hasType(field.getFieldObjectInspector(), types)) {
                return true;
            }
        }
        return false;
    }
    throw new IllegalArgumentException("Unknown object inspector type " + objectInspector);
}
 
Example 21
Source Project: presto   Source File: TestDataWritableWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * It writes the field value to the Parquet RecordConsumer. It detects the field type, and calls
 * the correct write function.
 *
 * @param value The writable object that contains the value.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the type schema.
 */
private void writeValue(Object value, ObjectInspector inspector, Type type)
{
    if (type.isPrimitive()) {
        checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE);
        writePrimitive(value, (PrimitiveObjectInspector) inspector);
    }
    else {
        GroupType groupType = type.asGroupType();
        OriginalType originalType = type.getOriginalType();

        if (OriginalType.LIST == originalType) {
            checkInspectorCategory(inspector, ObjectInspector.Category.LIST);
            if (singleLevelArray) {
                writeSingleLevelArray(value, (ListObjectInspector) inspector, groupType);
            }
            else {
                writeArray(value, (ListObjectInspector) inspector, groupType);
            }
        }
        else if (originalType != null && (originalType == OriginalType.MAP || originalType == OriginalType.MAP_KEY_VALUE)) {
            checkInspectorCategory(inspector, ObjectInspector.Category.MAP);
            writeMap(value, (MapObjectInspector) inspector, groupType);
        }
        else {
            checkInspectorCategory(inspector, ObjectInspector.Category.STRUCT);
            writeGroup(value, (StructObjectInspector) inspector, groupType);
        }
    }
}
 
Example 22
Source Project: transport   Source File: HiveMap.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
public HiveMap(Object object, ObjectInspector objectInspector, StdFactory stdFactory) {
  super(stdFactory);
  _object = object;
  _mapObjectInspector = (MapObjectInspector) objectInspector;
  _keyObjectInspector = _mapObjectInspector.getMapKeyObjectInspector();
  _valueObjectInspector = _mapObjectInspector.getMapValueObjectInspector();
}
 
Example 23
Source Project: transport   Source File: HiveFactory.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public StdMap createMap(StdType stdType) {
  MapObjectInspector mapObjectInspector = (MapObjectInspector) stdType.underlyingType();
  return new HiveMap(
      new HashMap(),
      ObjectInspectorFactory.getStandardMapObjectInspector(
          mapObjectInspector.getMapKeyObjectInspector(),
          mapObjectInspector.getMapValueObjectInspector()),
      this);
}
 
Example 24
Source Project: hive-third-functions   Source File: UDFMapElementAt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function map_element_at(map, key) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
        if (!arguments[0].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(0,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function map_element_at, but "
                            + "\"" + arguments[0].getTypeName() + "\" "
                            + "is found");
        }

    mapOI = (MapObjectInspector) arguments[0];
    keyOI = arguments[1];

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();

    // Check if map value type are of same value type
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, keyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + mapKeyOI.getTypeName() + "\""
                        + " expected at function map_element_at key, but "
                        + "\"" + keyOI.getTypeName() + "\""
                        + " is found");
    }

    return mapValueOI;
}
 
Example 25
Source Project: emr-dynamodb-connector   Source File: DynamoDBDataParser.java    License: Apache License 2.0 5 votes vote down vote up
public static Object getMapObject(Map<String, AttributeValue> data, ObjectInspector objectInspector) {
  MapObjectInspector mapOI = (MapObjectInspector) objectInspector;
  ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
  HiveDynamoDBType valueType = HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(mapValueOI);

  Map<String, Object> values = new HashMap<>();
  for (Map.Entry<String, AttributeValue> entry : data.entrySet()) {
    values.put(entry.getKey(), valueType.getHiveData(entry.getValue(), mapValueOI));
  }

  return values;
}
 
Example 26
Source Project: incubator-hivemall   Source File: JsonSerdeUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Serialize Hive objects as Text.
 */
@Nonnull
public static Text serialize(@Nullable final Object obj, @Nonnull final ObjectInspector oi,
        @Nullable final List<String> columnNames) throws SerDeException {
    final StringBuilder sb = new StringBuilder();
    switch (oi.getCategory()) {
        case STRUCT:
            StructObjectInspector soi = (StructObjectInspector) oi;
            serializeStruct(sb, obj, soi, columnNames);
            break;
        case LIST:
            ListObjectInspector loi = (ListObjectInspector) oi;
            serializeList(sb, obj, loi);
            break;
        case MAP:
            MapObjectInspector moi = (MapObjectInspector) oi;
            serializeMap(sb, obj, moi);
            break;
        case PRIMITIVE:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            serializePrimitive(sb, obj, poi);
            break;
        default:
            throw new SerDeException("Unknown type in ObjectInspector: " + oi.getCategory());
    }

    return new Text(sb.toString());
}
 
Example 27
Source Project: incubator-hivemall   Source File: JsonSerdeUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static void serializeMap(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nonnull final MapObjectInspector moi) throws SerDeException {
    ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
    ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
    Map<?, ?> omap = moi.getMap(obj);
    if (omap == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACE);
        boolean first = true;
        for (Object entry : omap.entrySet()) {
            if (first) {
                first = false;
            } else {
                sb.append(SerDeUtils.COMMA);
            }
            Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
            StringBuilder keyBuilder = new StringBuilder();
            buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
            String keyString = keyBuilder.toString().trim();
            if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
                appendWithQuotes(sb, keyString);
            } else {
                sb.append(keyString);
            }
            sb.append(SerDeUtils.COLON);
            buildJSONString(sb, e.getValue(), mapValueObjectInspector);
        }
        sb.append(SerDeUtils.RBRACE);
    }
}
 
Example 28
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi)
        throws UDFArgumentException {
    if (oi.getCategory() != Category.MAP) {
        throw new UDFArgumentException("Expected Map OI but was: " + oi);
    }
    return (MapObjectInspector) oi;
}
 
Example 29
Source Project: incubator-hivemall   Source File: HiveJsonStructReader.java    License: Apache License 2.0 5 votes vote down vote up
private Object parseDispatcher(JsonParser parser, ObjectInspector oi)
        throws JsonParseException, IOException, SerDeException {
    switch (oi.getCategory()) {
        case PRIMITIVE:
            return parsePrimitive(parser, (PrimitiveObjectInspector) oi);
        case LIST:
            return parseList(parser, (ListObjectInspector) oi);
        case STRUCT:
            return parseStruct(parser, (StructObjectInspector) oi);
        case MAP:
            return parseMap(parser, (MapObjectInspector) oi);
        default:
            throw new SerDeException("parsing of: " + oi.getCategory() + " is not handled");
    }
}
 
Example 30
Source Project: incubator-hivemall   Source File: HiveJsonStructReader.java    License: Apache License 2.0 5 votes vote down vote up
private Object parseMap(JsonParser parser, MapObjectInspector oi)
        throws IOException, SerDeException {
    if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
        parser.nextToken();
        return null;
    }

    Map<Object, Object> ret = new LinkedHashMap<>();

    if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
        throw new SerDeException("struct expected");
    }

    if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) {
        throw new SerDeException("map key must be a primitive");
    }
    PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector();
    ObjectInspector valOI = oi.getMapValueObjectInspector();

    JsonToken currentToken = parser.nextToken();
    while (currentToken != null && currentToken != JsonToken.END_OBJECT) {

        if (currentToken != JsonToken.FIELD_NAME) {
            throw new SerDeException("unexpected token: " + currentToken);
        }

        Object key = parseMapKey(parser, keyOI);
        Object val = parseDispatcher(parser, valOI);
        ret.put(key, val);

        currentToken = parser.getCurrentToken();
    }
    if (currentToken != null) {
        parser.nextToken();
    }
    return ret;
}