org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OrcFlowFileWriter.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        MapObjectInspector insp = (MapObjectInspector) inspector;
        // this sucks, but it will have to do until we can get a better
        // accessor in the MapObjectInspector.
        Map<?, ?> valueMap = insp.getMap(obj);
        lengths.write(valueMap.size());
        if (createBloomFilter) {
            bloomFilter.addLong(valueMap.size());
        }
        for (Map.Entry<?, ?> entry : valueMap.entrySet()) {
            childrenWriters[0].write(entry.getKey());
            childrenWriters[1].write(entry.getValue());
        }
    }
}
 
Example #2
Source File: MapTailNUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
            "map_tail_n only takes 2 arguments: map<object, object>, int");
    }
    if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentException("The first argument must be a map");
    }
    this.mapObjectInspector = (MapObjectInspector) arguments[0];
    if (!(arguments[1] instanceof IntObjectInspector)) {
        throw new UDFArgumentException("The second argument must be an int");
    }
    this.intObjectInspector = (IntObjectInspector) arguments[1];

    ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(
        mapObjectInspector.getMapKeyObjectInspector());
    ObjectInspector valueOI = mapObjectInspector.getMapValueObjectInspector();

    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}
 
Example #3
Source File: SlimUDTF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}
 
Example #4
Source File: JSONCDHSerDe.java    From bigdata-tutorial with Apache License 2.0 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example #5
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example #6
Source File: HiveORCVectorizedReader.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example #7
Source File: JSONSerDe.java    From searchanalytics-bigdata with MIT License 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 
Example #8
Source File: HiveMapObjectConverter.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
@Override
public Object convert(ObjectInspector objectInspector, Object o, TypeInfo odpsTypeInfo) {
  MapObjectInspector mapObjectInspector = (MapObjectInspector) objectInspector;
  ObjectInspector mapKeyObjectInspector = mapObjectInspector.getMapKeyObjectInspector();
  ObjectInspector mapValueObjectInspector = mapObjectInspector.getMapValueObjectInspector();
  TypeInfo mapKeyTypeInfo = ((MapTypeInfo) odpsTypeInfo).getKeyTypeInfo();
  TypeInfo mapValueTypeInfo = ((MapTypeInfo) odpsTypeInfo).getValueTypeInfo();

  Map map = mapObjectInspector.getMap(o);
  Map<Object, Object> newMap = new HashMap<>();
  for (Object k : map.keySet()) {
    Object v = map.get(k);
    newMap.put(HiveObjectConverter.convert(mapKeyObjectInspector, k, mapKeyTypeInfo),
        HiveObjectConverter.convert(mapValueObjectInspector, v, mapValueTypeInfo));
  }

  return newMap;
}
 
Example #9
Source File: HiveResolver.java    From pxf with Apache License 2.0 6 votes vote down vote up
private List<OneField> traverseMap(Object obj, MapObjectInspector moi)
        throws BadRecordException, IOException {
    List<OneField> complexRecord = new LinkedList<>();
    List<OneField> mapRecord = new LinkedList<>();
    ObjectInspector koi = moi.getMapKeyObjectInspector();
    ObjectInspector voi = moi.getMapValueObjectInspector();
    Map<?, ?> map = moi.getMap(obj);
    if (map == null) {
        throw new BadRecordException(
                "Illegal value NULL for Hive data type Map");
    } else if (map.isEmpty()) {
        traverseTuple(null, koi, complexRecord, true);
        traverseTuple(null, voi, complexRecord, true);
        addOneFieldToRecord(mapRecord, DataType.TEXT,
                HdfsUtilities.toString(complexRecord, mapkeyDelim));
    } else {
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            traverseTuple(entry.getKey(), koi, complexRecord, true);
            traverseTuple(entry.getValue(), voi, complexRecord, true);
            addOneFieldToRecord(mapRecord, DataType.TEXT,
                    HdfsUtilities.toString(complexRecord, mapkeyDelim));
            complexRecord.clear();
        }
    }
    return mapRecord;
}
 
Example #10
Source File: OrcFlowFileWriter.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable) throws IOException {
    super(columnId, inspector, writer, nullable);
    this.isDirectV2 = isNewWriteFormat(writer);
    MapObjectInspector insp = (MapObjectInspector) inspector;
    childrenWriters = new TreeWriter[2];
    childrenWriters[0] =
            createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
    childrenWriters[1] =
            createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
    lengths = createIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
    recordPosition(rowIndexPosition);
}
 
Example #11
Source File: WriterImpl.java    From hive-dwrf with Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
  long rawDataSize = 0;
  if (obj != null) {
    MapObjectInspector insp = (MapObjectInspector) inspector;
    // this sucks, but it will have to do until we can get a better
    // accessor in the MapObjectInspector.
    Map<?, ?> valueMap = insp.getMap(obj);
    // Don't use getMapSize(), it's inconsistent for some object inspectors
    int len = valueMap.size();
    lengths.write(len);
    for(Map.Entry<?, ?> entry: valueMap.entrySet()) {
      childrenWriters[0].write(entry.getKey());
      childrenWriters[1].write(entry.getValue());
      rawDataSize += childrenWriters[0].getRowRawDataSize();
      rawDataSize += childrenWriters[1].getRowRawDataSize();
    }
  }
  super.write(obj, rawDataSize);
}
 
Example #12
Source File: MapKeyValuesUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 1) {
        throw new UDFArgumentLengthException(
            "The function MAP_KEYS only accepts one argument.");
    } else if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentTypeException(0,
            "\"" + Category.MAP.toString().toLowerCase()
                    + "\" is expected at function MAP_KEYS, " + "but \""
                    + arguments[0].getTypeName() + "\" is found");
    }

    this.mapOI = (MapObjectInspector) arguments[0];

    List<String> structFieldNames = new ArrayList<String>();
    List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
    structFieldNames.add("key");
    structFieldObjectInspectors.add(mapOI.getMapKeyObjectInspector());
    structFieldNames.add("value");
    structFieldObjectInspectors.add(mapOI.getMapValueObjectInspector());

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames,
            structFieldObjectInspectors));
}
 
Example #13
Source File: OnehotEncodingUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #14
Source File: SerDeUtils.java    From presto with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static Block serializeObject(Type type, BlockBuilder builder, Object object, ObjectInspector inspector, boolean filterNullMapKeys)
{
    switch (inspector.getCategory()) {
        case PRIMITIVE:
            serializePrimitive(type, builder, object, (PrimitiveObjectInspector) inspector);
            return null;
        case LIST:
            return serializeList(type, builder, object, (ListObjectInspector) inspector);
        case MAP:
            return serializeMap(type, builder, object, (MapObjectInspector) inspector, filterNullMapKeys);
        case STRUCT:
            return serializeStruct(type, builder, object, (StructObjectInspector) inspector);
        case UNION:
            return serializeUnion(type, builder, object, (UnionObjectInspector) inspector);
    }
    throw new RuntimeException("Unknown object inspector category: " + inspector.getCategory());
}
 
Example #15
Source File: CacheableObjectInspectorConverters.java    From transport with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public MapConverter(ObjectInspector inputOI, SettableMapObjectInspector outputOI) {
  if (inputOI instanceof MapObjectInspector) {
    this.inputOI = (MapObjectInspector) inputOI;
    this.outputOI = outputOI;
    inputKeyOI = this.inputOI.getMapKeyObjectInspector();
    outputKeyOI = outputOI.getMapKeyObjectInspector();
    inputValueOI = this.inputOI.getMapValueObjectInspector();
    outputValueOI = outputOI.getMapValueObjectInspector();
    keyConverter = getConverter(inputKeyOI, outputKeyOI);
    valueConverter = getConverter(inputValueOI, outputValueOI);
  } else if (!(inputOI instanceof VoidObjectInspector)) {
    throw new UnsupportedOperationException(
        "Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName()
            + "not supported yet.");
  }
}
 
Example #16
Source File: WriterImpl.java    From hive-dwrf with Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable, Configuration conf,
              boolean useVInts, boolean lowMemoryMode,
              MemoryEstimate memoryEstimate) throws IOException {
  super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
  MapObjectInspector insp = (MapObjectInspector) inspector;
  childrenWriters = new TreeWriter[2];
  childrenWriters[0] =
    createTreeWriter(insp.getMapKeyObjectInspector(), writer, true, conf, useVInts,
        lowMemoryMode, memoryEstimate);
  childrenWriters[1] =
    createTreeWriter(insp.getMapValueObjectInspector(), writer, true, conf, useVInts,
        lowMemoryMode, memoryEstimate);
  lengths =
    new RunLengthIntegerWriter(writer.createStream(columnId,
        OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
  recordPosition(rowIndexPosition);
}
 
Example #17
Source File: OrcFlowFileWriter.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        MapObjectInspector insp = (MapObjectInspector) inspector;
        // this sucks, but it will have to do until we can get a better
        // accessor in the MapObjectInspector.
        Map<?, ?> valueMap = insp.getMap(obj);
        lengths.write(valueMap.size());
        if (createBloomFilter) {
            bloomFilter.addLong(valueMap.size());
        }
        for (Map.Entry<?, ?> entry : valueMap.entrySet()) {
            childrenWriters[0].write(entry.getKey());
            childrenWriters[1].write(entry.getValue());
        }
    }
}
 
Example #18
Source File: OrcFlowFileWriter.java    From nifi with Apache License 2.0 6 votes vote down vote up
MapTreeWriter(int columnId,
              ObjectInspector inspector,
              StreamFactory writer,
              boolean nullable) throws IOException {
    super(columnId, inspector, writer, nullable);
    this.isDirectV2 = isNewWriteFormat(writer);
    MapObjectInspector insp = (MapObjectInspector) inspector;
    childrenWriters = new TreeWriter[2];
    childrenWriters[0] =
            createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
    childrenWriters[1] =
            createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
    lengths = createIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
    recordPosition(rowIndexPosition);
}
 
Example #19
Source File: HiveJsonStructReader.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
private Object parseMap(JsonParser parser, MapObjectInspector oi)
        throws IOException, SerDeException {
    if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
        parser.nextToken();
        return null;
    }

    Map<Object, Object> ret = new LinkedHashMap<>();

    if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
        throw new SerDeException("struct expected");
    }

    if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) {
        throw new SerDeException("map key must be a primitive");
    }
    PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector();
    ObjectInspector valOI = oi.getMapValueObjectInspector();

    JsonToken currentToken = parser.nextToken();
    while (currentToken != null && currentToken != JsonToken.END_OBJECT) {

        if (currentToken != JsonToken.FIELD_NAME) {
            throw new SerDeException("unexpected token: " + currentToken);
        }

        Object key = parseMapKey(parser, keyOI);
        Object val = parseDispatcher(parser, valOI);
        ret.put(key, val);

        currentToken = parser.getCurrentToken();
    }
    if (currentToken != null) {
        parser.nextToken();
    }
    return ret;
}
 
Example #20
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Nonnull
public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi)
        throws UDFArgumentException {
    if (oi.getCategory() != Category.MAP) {
        throw new UDFArgumentException("Expected Map OI but was: " + oi);
    }
    return (MapObjectInspector) oi;
}
 
Example #21
Source File: MergeMapsUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
private static void putIntoSet(@Nonnull final Map<?, ?> m,
        @Nonnull final Map<Object, Object> dst, @Nonnull final MapObjectInspector mapOI) {
    final ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
    final ObjectInspector valueOI = mapOI.getMapValueObjectInspector();

    for (Map.Entry<?, ?> e : m.entrySet()) {
        Object k = e.getKey();
        Object v = e.getValue();
        Object keyCopy = ObjectInspectorUtils.copyToStandardObject(k, keyOI);
        Object valCopy = ObjectInspectorUtils.copyToStandardObject(v, valueOI);
        dst.put(keyCopy, valCopy);
    }
}
 
Example #22
Source File: JsonSerdeUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
private static void serializeMap(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nonnull final MapObjectInspector moi) throws SerDeException {
    ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
    ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
    Map<?, ?> omap = moi.getMap(obj);
    if (omap == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACE);
        boolean first = true;
        for (Object entry : omap.entrySet()) {
            if (first) {
                first = false;
            } else {
                sb.append(SerDeUtils.COMMA);
            }
            Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
            StringBuilder keyBuilder = new StringBuilder();
            buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
            String keyString = keyBuilder.toString().trim();
            if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
                appendWithQuotes(sb, keyString);
            } else {
                sb.append(keyString);
            }
            sb.append(SerDeUtils.COLON);
            buildJSONString(sb, e.getValue(), mapValueObjectInspector);
        }
        sb.append(SerDeUtils.RBRACE);
    }
}
 
Example #23
Source File: ParquetHiveSerDe.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private Writable createMap(final Object obj, final MapObjectInspector inspector)
    throws SerDeException {
  final Map<?, ?> sourceMap = inspector.getMap(obj);
  final ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
  final ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
  final List<ArrayWritable> array = new ArrayList<ArrayWritable>();

  if (sourceMap != null) {
    for (final Entry<?, ?> keyValue : sourceMap.entrySet()) {
      final Writable key = createObject(keyValue.getKey(), keyInspector);
      final Writable value = createObject(keyValue.getValue(), valueInspector);
      if (key != null) {
        Writable[] arr = new Writable[2];
        arr[0] = key;
        arr[1] = value;
        array.add(new ArrayWritable(Writable.class, arr));
      }
    }
  }
  if (array.size() > 0) {
    final ArrayWritable subArray = new ArrayWritable(ArrayWritable.class,
        array.toArray(new ArrayWritable[array.size()]));
    return new ArrayWritable(Writable.class, new Writable[] {subArray});
  } else {
    return null;
  }
}
 
Example #24
Source File: JsonSerdeUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize Hive objects as Text.
 */
@Nonnull
public static Text serialize(@Nullable final Object obj, @Nonnull final ObjectInspector oi,
        @Nullable final List<String> columnNames) throws SerDeException {
    final StringBuilder sb = new StringBuilder();
    switch (oi.getCategory()) {
        case STRUCT:
            StructObjectInspector soi = (StructObjectInspector) oi;
            serializeStruct(sb, obj, soi, columnNames);
            break;
        case LIST:
            ListObjectInspector loi = (ListObjectInspector) oi;
            serializeList(sb, obj, loi);
            break;
        case MAP:
            MapObjectInspector moi = (MapObjectInspector) oi;
            serializeMap(sb, obj, moi);
            break;
        case PRIMITIVE:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            serializePrimitive(sb, obj, poi);
            break;
        default:
            throw new SerDeException("Unknown type in ObjectInspector: " + oi.getCategory());
    }

    return new Text(sb.toString());
}
 
Example #25
Source File: MapGetUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException("map_get accepts exactly 2 arguments.");
    }

    if (arguments[0] instanceof MapObjectInspector) {
        this.mapOI = (MapObjectInspector) arguments[0];
    } else {
        throw new UDFArgumentTypeException(0,
            "\"map\" is expected for the first argument, but \"" + arguments[0].getTypeName()
                    + "\" is found");
    }

    // index has to be a primitive
    if (!(arguments[1] instanceof PrimitiveObjectInspector)) {
        throw new UDFArgumentTypeException(1,
            "Primitive Type is expected but " + arguments[1].getTypeName() + "\" is found");
    }

    PrimitiveObjectInspector inputOI = (PrimitiveObjectInspector) arguments[1];
    ObjectInspector indexOI =
            ObjectInspectorConverters.getConvertedOI(inputOI, mapOI.getMapKeyObjectInspector());
    this.converter = ObjectInspectorConverters.getConverter(inputOI, indexOI);

    return mapOI.getMapValueObjectInspector();
}
 
Example #26
Source File: UDAFToOrderedMap.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        this.inputValueOI = argOIs[1];
        this.sizeOI = HiveUtils.asIntegerOI(argOIs[2]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) argOIs[0];
        this.internalMergeOI = soi;

        this.partialMapField = soi.getStructFieldRef("partialMap");
        // re-extract input key/value OIs
        MapObjectInspector partialMapOI =
                (MapObjectInspector) partialMapField.getFieldObjectInspector();
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
            partialMapOI.getMapKeyObjectInspector());
        this.inputValueOI = partialMapOI.getMapValueObjectInspector();

        this.partialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));

        this.sizeField = soi.getStructFieldRef("size");
        this.sizeOI = (PrimitiveObjectInspector) sizeField.getFieldObjectInspector();
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI(inputKeyOI, inputValueOI);
    } else {// terminate
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
    }
    return outputOI;
}
 
Example #27
Source File: HiveJsonStructReader.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
private Object parseDispatcher(JsonParser parser, ObjectInspector oi)
        throws JsonParseException, IOException, SerDeException {
    switch (oi.getCategory()) {
        case PRIMITIVE:
            return parsePrimitive(parser, (PrimitiveObjectInspector) oi);
        case LIST:
            return parseList(parser, (ListObjectInspector) oi);
        case STRUCT:
            return parseStruct(parser, (StructObjectInspector) oi);
        case MAP:
            return parseMap(parser, (MapObjectInspector) oi);
        default:
            throw new SerDeException("parsing of: " + oi.getCategory() + " is not handled");
    }
}
 
Example #28
Source File: DynamoDBDataParser.java    From emr-dynamodb-connector with Apache License 2.0 5 votes vote down vote up
public static Object getMapObject(Map<String, AttributeValue> data, ObjectInspector objectInspector) {
  MapObjectInspector mapOI = (MapObjectInspector) objectInspector;
  ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();
  HiveDynamoDBType valueType = HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(mapValueOI);

  Map<String, Object> values = new HashMap<>();
  for (Map.Entry<String, AttributeValue> entry : data.entrySet()) {
    values.put(entry.getKey(), valueType.getHiveData(entry.getValue(), mapValueOI));
  }

  return values;
}
 
Example #29
Source File: UDFMapElementAt.java    From hive-third-functions with Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function map_element_at(map, key) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
        if (!arguments[0].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(0,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function map_element_at, but "
                            + "\"" + arguments[0].getTypeName() + "\" "
                            + "is found");
        }

    mapOI = (MapObjectInspector) arguments[0];
    keyOI = arguments[1];

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector mapValueOI = mapOI.getMapValueObjectInspector();

    // Check if map value type are of same value type
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, keyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + mapKeyOI.getTypeName() + "\""
                        + " expected at function map_element_at key, but "
                        + "\"" + keyOI.getTypeName() + "\""
                        + " is found");
    }

    return mapValueOI;
}
 
Example #30
Source File: MapProcessor.java    From hive-solr with Apache License 2.0 5 votes vote down vote up
public static void resolve(boolean enableFieldMapping, LWDocument doc, String docFieldName, Object data,
  StructField structField, StructObjectInspector inspector) {
  MapObjectInspector moi = (MapObjectInspector) structField.getFieldObjectInspector();
  Object mapValue = inspector.getStructFieldData(data, structField);
  Map<Object, Object> map = (Map<Object, Object>) copyToStandardJavaObject(mapValue, moi);
  Map<String, Object> entries = new HashMap<>();
  resolveMap(enableFieldMapping, entries, docFieldName, map);

  for (Map.Entry<String, Object> entry : entries.entrySet()) {
    doc.addField(entry.getKey(), entry.getValue());
  }

  entries.clear();
}