Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: UDAFToMap.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        inputValueOI = argOIs[1];
    } else {// from partial aggregation
        internalMergeOI = (StandardMapObjectInspector) argOIs[0];
        inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
            internalMergeOI.getMapKeyObjectInspector());
        inputValueOI = internalMergeOI.getMapValueObjectInspector();
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
        ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
}

Example 2

Source File: MergeMapsUDAF.java From incubator-hivemall with Apache License 2.0

6 votes

public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    Preconditions.checkArgument(parameters.length == 1);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputMapOI = HiveUtils.asMapOI(parameters[0]);
        this.inputKeyOI = inputMapOI.getMapKeyObjectInspector();
        this.inputValOI = inputMapOI.getMapValueObjectInspector();
    } else {// from partial aggregation
        this.mergeMapOI = HiveUtils.asMapOI(parameters[0]);
        this.inputKeyOI = mergeMapOI.getMapKeyObjectInspector();
        this.inputValOI = mergeMapOI.getMapValueObjectInspector();
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
        ObjectInspectorUtils.getStandardObjectInspector(inputValOI));
}

Example 3

Source File: OnehotEncodingUDAF.java From incubator-hivemall with Apache License 2.0

6 votes

@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

Example 4

Source File: UDFMapConcatTest.java From hive-third-functions with Apache License 2.0

6 votes

@Test
public void testMapConcat() throws Exception {
    UDFMapConcat udf = new UDFMapConcat();
    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    LinkedHashMap<String, String> leftMap = Maps.newLinkedHashMap();
    leftMap.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "13"));
    LinkedHashMap<String, String> rightMap = Maps.newLinkedHashMap();
    rightMap.putAll(ImmutableMap.<String, String>of("key3", "21", "key4", "22", "key5", "23"));

    DeferredObject leftMapObj = new DeferredJavaObject(leftMap);
    DeferredObject rightMapObj = new DeferredJavaObject(rightMap);
    DeferredObject[] args = {leftMapObj, rightMapObj};
    LinkedHashMap<String, String> output = (LinkedHashMap<String, String>) udf.evaluate(args);
    LinkedHashMap<String, String> expect = Maps.newLinkedHashMap();
    expect.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "21", "key4", "22", "key5", "23"));

    Assert.assertEquals("map_concat() test", true, MapUtils.mapEquals(output, expect));
}

Example 5

Source File: MapTailNUDF.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
            "map_tail_n only takes 2 arguments: map<object, object>, int");
    }
    if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentException("The first argument must be a map");
    }
    this.mapObjectInspector = (MapObjectInspector) arguments[0];
    if (!(arguments[1] instanceof IntObjectInspector)) {
        throw new UDFArgumentException("The second argument must be an int");
    }
    this.intObjectInspector = (IntObjectInspector) arguments[1];

    ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(
        mapObjectInspector.getMapKeyObjectInspector());
    ObjectInspector valueOI = mapObjectInspector.getMapValueObjectInspector();

    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}

Example 6

Source File: UDFStringSplitToMultimap.java From hive-third-functions with Apache License 2.0

6 votes

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_multimap, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}

Example 7

Source File: UDFMathCosineSimilarityTest.java From hive-third-functions with Apache License 2.0

5 votes

public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException {
    UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity();

    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap);
    GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap);
    GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj};
    DoubleWritable output = (DoubleWritable) udf.evaluate(args);
    return output == null ? null : output.get();
}

Example 8

Source File: MajorityVoteUDAF.java From incubator-hivemall with Apache License 2.0

5 votes

@Override
public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs)
        throws HiveException {
    assert (argOIs.length == 1);
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.keyInputOI = asPrimitiveObjectInspector(argOIs[0]);
    } else {// from partial aggregation
        this.partialOI = (StandardMapObjectInspector) argOIs[0];
        this.keyInputOI = asPrimitiveObjectInspector(partialOI.getMapKeyObjectInspector());
        this.counterInputOI = asLongOI(partialOI.getMapValueObjectInspector());
    }
    this.keyOutputOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
        keyInputOI.getTypeInfo());

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOutputOI,
            PrimitiveObjectInspectorFactory.javaLongObjectInspector);
    } else {// terminate
        outputOI = keyOutputOI;
    }
    return outputOI;
}

Example 9

Source File: HiveInspectors.java From flink with Apache License 2.0

5 votes

private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}

Example 10

Source File: HiveTypeSystem.java From transport with BSD 2-Clause "Simplified" License

5 votes

@Override
protected ObjectInspector createMapType(ObjectInspector keyType, ObjectInspector valueType) {
  return ObjectInspectorFactory.getStandardMapObjectInspector(
      keyType,
      valueType
  );
}

Example 11

Source File: UDFMathCosineSimilarity.java From hive-third-functions with Apache License 2.0

4 votes

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function cosine_similarity(map, map) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
    for (int i = 0; i < 2; i++) {
        if (!arguments[i].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(i,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function cosine_similarity, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    leftMapOI = (MapObjectInspector) arguments[0];
    rightMapOI = (MapObjectInspector) arguments[1];

    ObjectInspector leftMapKeyOI = leftMapOI.getMapKeyObjectInspector();
    ObjectInspector leftMapValueOI = leftMapOI.getMapValueObjectInspector();
    ObjectInspector rightMapKeyOI = rightMapOI.getMapKeyObjectInspector();
    ObjectInspector rightMapValueOI = rightMapOI.getMapValueObjectInspector();

    // Check if two map are of same key and value type
    if (!ObjectInspectorUtils.compareTypes(leftMapKeyOI, rightMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + rightMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, leftMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(leftMapValueOI, rightMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapValueOI.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + rightMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, leftMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + leftMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(leftMapKeyOI, leftMapValueOI);
}

Example 12

Source File: RcFileTester.java From presto with Apache License 2.0

4 votes

private static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return javaStringObjectInspector;
    }
    if (type.equals(VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (type instanceof ArrayType) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (type instanceof MapType) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (type instanceof RowType) {
        return getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toList()),
                type.getTypeParameters().stream()
                        .map(RcFileTester::getJavaObjectInspector)
                        .collect(toList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}

Example 13

Source File: StandardObjectInspectorFactory.java From transport with BSD 2-Clause "Simplified" License

4 votes

public static ObjectInspector map(ObjectInspector keyObjectInspector, ObjectInspector valueObjectInspector) {
  return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
}

Example 14

Source File: AUCUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("indexScore");
    fieldOIs.add(writableDoubleObjectInspector);
    fieldNames.add("area");
    fieldOIs.add(writableDoubleObjectInspector);
    fieldNames.add("fp");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("tp");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("fpPrev");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("tpPrev");
    fieldOIs.add(writableLongObjectInspector);

    MapObjectInspector areaPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaDoubleObjectInspector);
    fieldNames.add("areaPartialMap");
    fieldOIs.add(areaPartialMapOI);

    MapObjectInspector fpPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("fpPartialMap");
    fieldOIs.add(fpPartialMapOI);

    MapObjectInspector tpPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("tpPartialMap");
    fieldOIs.add(tpPartialMapOI);

    MapObjectInspector fpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("fpPrevPartialMap");
    fieldOIs.add(fpPrevPartialMapOI);

    MapObjectInspector tpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("tpPrevPartialMap");
    fieldOIs.add(tpPrevPartialMapOI);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

Example 15

Source File: AUCUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
@Override
public void merge(AggregationBuffer agg, Object partial) throws HiveException {
    if (partial == null) {
        return;
    }

    Object indexScoreObj = internalMergeOI.getStructFieldData(partial, indexScoreField);
    Object areaObj = internalMergeOI.getStructFieldData(partial, areaField);
    Object fpObj = internalMergeOI.getStructFieldData(partial, fpField);
    Object tpObj = internalMergeOI.getStructFieldData(partial, tpField);
    Object fpPrevObj = internalMergeOI.getStructFieldData(partial, fpPrevField);
    Object tpPrevObj = internalMergeOI.getStructFieldData(partial, tpPrevField);
    Object areaPartialMapObj =
            internalMergeOI.getStructFieldData(partial, areaPartialMapField);
    Object fpPartialMapObj = internalMergeOI.getStructFieldData(partial, fpPartialMapField);
    Object tpPartialMapObj = internalMergeOI.getStructFieldData(partial, tpPartialMapField);
    Object fpPrevPartialMapObj =
            internalMergeOI.getStructFieldData(partial, fpPrevPartialMapField);
    Object tpPrevPartialMapObj =
            internalMergeOI.getStructFieldData(partial, tpPrevPartialMapField);

    double indexScore = writableDoubleObjectInspector.get(indexScoreObj);
    double area = writableDoubleObjectInspector.get(areaObj);
    long fp = writableLongObjectInspector.get(fpObj);
    long tp = writableLongObjectInspector.get(tpObj);
    long fpPrev = writableLongObjectInspector.get(fpPrevObj);
    long tpPrev = writableLongObjectInspector.get(tpPrevObj);

    StandardMapObjectInspector ddMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaDoubleObjectInspector);
    StandardMapObjectInspector dlMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);

    Map<Double, Double> areaPartialMap = (Map<Double, Double>) ddMapOI.getMap(
        HiveUtils.castLazyBinaryObject(areaPartialMapObj));
    Map<Double, Long> fpPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(fpPartialMapObj));
    Map<Double, Long> tpPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(tpPartialMapObj));
    Map<Double, Long> fpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(fpPrevPartialMapObj));
    Map<Double, Long> tpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(tpPrevPartialMapObj));

    ClassificationAUCAggregationBuffer myAggr = (ClassificationAUCAggregationBuffer) agg;
    myAggr.merge(indexScore, area, fp, tp, fpPrev, tpPrev, areaPartialMap, fpPartialMap,
        tpPartialMap, fpPrevPartialMap, tpPrevPartialMap);
}

Example 16

Source File: SlimUDTFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Test
public void testAllSamples() throws HiveException {
    SlimUDTF slim = new SlimUDTF();
    ObjectInspector itemIOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector itemJOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;

    ObjectInspector itemIRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
    ObjectInspector itemJRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
    ObjectInspector topKRatesOfIOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaIntObjectInspector,
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector));
    ObjectInspector optionArgumentOI = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-l2 0.01 -l1 0.01");

    ObjectInspector[] argOIs =
            {itemIOI, itemIRatesOI, topKRatesOfIOI, itemJOI, itemJRatesOI, optionArgumentOI};

    slim.initialize(argOIs);
    int numUser = 4;
    int numItem = 5;

    float[][] data = {{1.f, 4.f, 0.f, 0.f, 0.f}, {0.f, 3.f, 0.f, 1.f, 2.f},
            {2.f, 2.f, 0.f, 0.f, 3.f}, {0.f, 1.f, 1.f, 0.f, 0.f}};

    for (int i = 0; i < numItem; i++) {
        Map<Integer, Float> Ri = new HashMap<>();
        for (int u = 0; u < numUser; u++) {
            if (data[u][i] != 0.) {
                Ri.put(u, data[u][i]);
            }
        }

        // most similar data
        Map<Integer, Map<Integer, Float>> knnRatesOfI = new HashMap<>();
        for (int u = 0; u < numUser; u++) {
            Map<Integer, Float> Ru = new HashMap<>();
            for (int k = 0; k < numItem; k++) {
                if (k == i)
                    continue;
                Ru.put(k, data[u][k]);
            }
            knnRatesOfI.put(u, Ru);
        }

        for (int j = 0; j < numItem; j++) {
            if (i == j)
                continue;
            Map<Integer, Float> Rj = new HashMap<>();
            for (int u = 0; u < numUser; u++) {
                if (data[u][j] != 0.) {
                    Rj.put(u, data[u][j]);
                }
            }

            Object[] args = {i, Ri, knnRatesOfI, j, Rj};
            slim.process(args);
        }
    }
    slim.finalizeTraining();
}

Example 17

Source File: AUCUDAFTest.java From incubator-hivemall with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
    auc = new AUCUDAF();

    inputOIs = new ObjectInspector[] {
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.DOUBLE),
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.INT)};

    evaluator = auc.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("indexScore");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("area");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("fp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPrev");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPrev");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    MapObjectInspector areaPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("areaPartialMap");
    fieldOIs.add(areaPartialMapOI);

    MapObjectInspector fpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPartialMap");
    fieldOIs.add(fpPartialMapOI);

    MapObjectInspector tpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPartialMap");
    fieldOIs.add(tpPartialMapOI);

    MapObjectInspector fpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPrevPartialMap");
    fieldOIs.add(fpPrevPartialMapOI);

    MapObjectInspector tpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPrevPartialMap");
    fieldOIs.add(tpPrevPartialMapOI);

    partialOI = new ObjectInspector[2];
    partialOI[0] =
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);

    agg = (AUCUDAF.ClassificationAUCAggregationBuffer) evaluator.getNewAggregationBuffer();
}

Example 18

Source File: LDAPredictUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        processOptions(parameters);
        this.wordOI = HiveUtils.asStringOI(parameters[0]);
        this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
        this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
        this.lambdaOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.wcListField = soi.getStructFieldRef("wcList");
        this.lambdaMapField = soi.getStructFieldRef("lambdaMap");
        this.topicsOptionField = soi.getStructFieldRef("topics");
        this.alphaOptionField = soi.getStructFieldRef("alpha");
        this.deltaOptionField = soi.getStructFieldRef("delta");
        this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
        this.lambdaMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.lambdaMapValueElemOI =
                PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.lambdaMapValueOI =
                ObjectInspectorFactory.getStandardListObjectInspector(lambdaMapValueElemOI);
        this.lambdaMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            lambdaMapKeyOI, lambdaMapValueOI);
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {
        final ArrayList<String> fieldNames = new ArrayList<String>();
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
    }
    return outputOI;
}

Example 19

Source File: PLSAPredictUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        processOptions(parameters);
        this.wordOI = HiveUtils.asStringOI(parameters[0]);
        this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
        this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
        this.probOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.wcListField = soi.getStructFieldRef("wcList");
        this.probMapField = soi.getStructFieldRef("probMap");
        this.topicsOptionField = soi.getStructFieldRef("topics");
        this.alphaOptionField = soi.getStructFieldRef("alpha");
        this.deltaOptionField = soi.getStructFieldRef("delta");
        this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
        this.probMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.probMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.probMapValueOI =
                ObjectInspectorFactory.getStandardListObjectInspector(probMapValueElemOI);
        this.probMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(probMapKeyOI,
            probMapValueOI);
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {
        final ArrayList<String> fieldNames = new ArrayList<String>();
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
    }
    return outputOI;
}

Example 20

Source File: HiveWriteUtils.java From presto with Apache License 2.0

4 votes

public static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BooleanType.BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BigintType.BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(IntegerType.INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TinyintType.TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(RealType.REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return writableStringObjectInspector;
    }
    if (type instanceof CharType) {
        return writableHiveCharObjectInspector;
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DateType.DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (isArrayType(type)) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (isMapType(type)) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (isRowType(type)) {
        return ObjectInspectorFactory.getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toImmutableList()),
                type.getTypeParameters().stream()
                        .map(HiveWriteUtils::getJavaObjectInspector)
                        .collect(toImmutableList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}