Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_multimap, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 
Example 2
Source Project: incubator-hivemall   File: MapTailNUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
            "map_tail_n only takes 2 arguments: map<object, object>, int");
    }
    if (!(arguments[0] instanceof MapObjectInspector)) {
        throw new UDFArgumentException("The first argument must be a map");
    }
    this.mapObjectInspector = (MapObjectInspector) arguments[0];
    if (!(arguments[1] instanceof IntObjectInspector)) {
        throw new UDFArgumentException("The second argument must be an int");
    }
    this.intObjectInspector = (IntObjectInspector) arguments[1];

    ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(
        mapObjectInspector.getMapKeyObjectInspector());
    ObjectInspector valueOI = mapObjectInspector.getMapValueObjectInspector();

    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}
 
Example 3
@Test
public void testMapConcat() throws Exception {
    UDFMapConcat udf = new UDFMapConcat();
    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    LinkedHashMap<String, String> leftMap = Maps.newLinkedHashMap();
    leftMap.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "13"));
    LinkedHashMap<String, String> rightMap = Maps.newLinkedHashMap();
    rightMap.putAll(ImmutableMap.<String, String>of("key3", "21", "key4", "22", "key5", "23"));

    DeferredObject leftMapObj = new DeferredJavaObject(leftMap);
    DeferredObject rightMapObj = new DeferredJavaObject(rightMap);
    DeferredObject[] args = {leftMapObj, rightMapObj};
    LinkedHashMap<String, String> output = (LinkedHashMap<String, String>) udf.evaluate(args);
    LinkedHashMap<String, String> expect = Maps.newLinkedHashMap();
    expect.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "21", "key4", "22", "key5", "23"));

    Assert.assertEquals("map_concat() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 4
@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 5
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    Preconditions.checkArgument(parameters.length == 1);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputMapOI = HiveUtils.asMapOI(parameters[0]);
        this.inputKeyOI = inputMapOI.getMapKeyObjectInspector();
        this.inputValOI = inputMapOI.getMapValueObjectInspector();
    } else {// from partial aggregation
        this.mergeMapOI = HiveUtils.asMapOI(parameters[0]);
        this.inputKeyOI = mergeMapOI.getMapKeyObjectInspector();
        this.inputValOI = mergeMapOI.getMapValueObjectInspector();
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
        ObjectInspectorUtils.getStandardObjectInspector(inputValOI));
}
 
Example 6
Source Project: incubator-hivemall   File: UDAFToMap.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        inputValueOI = argOIs[1];
    } else {// from partial aggregation
        internalMergeOI = (StandardMapObjectInspector) argOIs[0];
        inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
            internalMergeOI.getMapKeyObjectInspector());
        inputValueOI = internalMergeOI.getMapValueObjectInspector();
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
        ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
}
 
Example 7
@Override
protected ObjectInspector createMapType(ObjectInspector keyType, ObjectInspector valueType) {
  return ObjectInspectorFactory.getStandardMapObjectInspector(
      keyType,
      valueType
  );
}
 
Example 8
Source Project: flink   File: HiveInspectors.java    License: Apache License 2.0 5 votes vote down vote up
private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}
 
Example 9
public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException {
    UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity();

    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap);
    GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap);
    GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj};
    DoubleWritable output = (DoubleWritable) udf.evaluate(args);
    return output == null ? null : output.get();
}
 
Example 10
@Override
public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs)
        throws HiveException {
    assert (argOIs.length == 1);
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.keyInputOI = asPrimitiveObjectInspector(argOIs[0]);
    } else {// from partial aggregation
        this.partialOI = (StandardMapObjectInspector) argOIs[0];
        this.keyInputOI = asPrimitiveObjectInspector(partialOI.getMapKeyObjectInspector());
        this.counterInputOI = asLongOI(partialOI.getMapValueObjectInspector());
    }
    this.keyOutputOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
        keyInputOI.getTypeInfo());

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOutputOI,
            PrimitiveObjectInspectorFactory.javaLongObjectInspector);
    } else {// terminate
        outputOI = keyOutputOI;
    }
    return outputOI;
}
 
Example 11
Source Project: presto   File: RcFileTester.java    License: Apache License 2.0 4 votes vote down vote up
private static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return javaStringObjectInspector;
    }
    if (type.equals(VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (type instanceof ArrayType) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (type instanceof MapType) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (type instanceof RowType) {
        return getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toList()),
                type.getTypeParameters().stream()
                        .map(RcFileTester::getJavaObjectInspector)
                        .collect(toList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example 12
Source Project: presto   File: HiveWriteUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getJavaObjectInspector(Type type)
{
    if (type.equals(BooleanType.BOOLEAN)) {
        return javaBooleanObjectInspector;
    }
    if (type.equals(BigintType.BIGINT)) {
        return javaLongObjectInspector;
    }
    if (type.equals(IntegerType.INTEGER)) {
        return javaIntObjectInspector;
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return javaShortObjectInspector;
    }
    if (type.equals(TinyintType.TINYINT)) {
        return javaByteObjectInspector;
    }
    if (type.equals(RealType.REAL)) {
        return javaFloatObjectInspector;
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return javaDoubleObjectInspector;
    }
    if (type instanceof VarcharType) {
        return writableStringObjectInspector;
    }
    if (type instanceof CharType) {
        return writableHiveCharObjectInspector;
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return javaByteArrayObjectInspector;
    }
    if (type.equals(DateType.DATE)) {
        return javaDateObjectInspector;
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return javaTimestampObjectInspector;
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()));
    }
    if (isArrayType(type)) {
        return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0)));
    }
    if (isMapType(type)) {
        ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0));
        ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1));
        return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
    }
    if (isRowType(type)) {
        return ObjectInspectorFactory.getStandardStructObjectInspector(
                type.getTypeSignature().getParameters().stream()
                        .map(parameter -> parameter.getNamedTypeSignature().getName().get())
                        .collect(toImmutableList()),
                type.getTypeParameters().stream()
                        .map(HiveWriteUtils::getJavaObjectInspector)
                        .collect(toImmutableList()));
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
 
Example 13
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        processOptions(parameters);
        this.wordOI = HiveUtils.asStringOI(parameters[0]);
        this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
        this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
        this.lambdaOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.wcListField = soi.getStructFieldRef("wcList");
        this.lambdaMapField = soi.getStructFieldRef("lambdaMap");
        this.topicsOptionField = soi.getStructFieldRef("topics");
        this.alphaOptionField = soi.getStructFieldRef("alpha");
        this.deltaOptionField = soi.getStructFieldRef("delta");
        this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
        this.lambdaMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.lambdaMapValueElemOI =
                PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.lambdaMapValueOI =
                ObjectInspectorFactory.getStandardListObjectInspector(lambdaMapValueElemOI);
        this.lambdaMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            lambdaMapKeyOI, lambdaMapValueOI);
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {
        final ArrayList<String> fieldNames = new ArrayList<String>();
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
    }
    return outputOI;
}
 
Example 14
Source Project: incubator-hivemall   File: AUCUDAFTest.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
    auc = new AUCUDAF();

    inputOIs = new ObjectInspector[] {
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.DOUBLE),
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.INT)};

    evaluator = auc.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    fieldNames.add("indexScore");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("area");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("fp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tp");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPrev");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPrev");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);

    MapObjectInspector areaPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    fieldNames.add("areaPartialMap");
    fieldOIs.add(areaPartialMapOI);

    MapObjectInspector fpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPartialMap");
    fieldOIs.add(fpPartialMapOI);

    MapObjectInspector tpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPartialMap");
    fieldOIs.add(tpPartialMapOI);

    MapObjectInspector fpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("fpPrevPartialMap");
    fieldOIs.add(fpPrevPartialMapOI);

    MapObjectInspector tpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    fieldNames.add("tpPrevPartialMap");
    fieldOIs.add(tpPrevPartialMapOI);

    partialOI = new ObjectInspector[2];
    partialOI[0] =
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);

    agg = (AUCUDAF.ClassificationAUCAggregationBuffer) evaluator.getNewAggregationBuffer();
}
 
Example 15
public static ObjectInspector map(ObjectInspector keyObjectInspector, ObjectInspector valueObjectInspector) {
  return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
}
 
Example 16
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function cosine_similarity(map, map) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of category LIST
    for (int i = 0; i < 2; i++) {
        if (!arguments[i].getCategory().equals(ObjectInspector.Category.MAP)) {
            throw new UDFArgumentTypeException(i,
                    "\"" + serdeConstants.MAP_TYPE_NAME + "\" "
                            + "expected at function cosine_similarity, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    leftMapOI = (MapObjectInspector) arguments[0];
    rightMapOI = (MapObjectInspector) arguments[1];

    ObjectInspector leftMapKeyOI = leftMapOI.getMapKeyObjectInspector();
    ObjectInspector leftMapValueOI = leftMapOI.getMapValueObjectInspector();
    ObjectInspector rightMapKeyOI = rightMapOI.getMapKeyObjectInspector();
    ObjectInspector rightMapValueOI = rightMapOI.getMapValueObjectInspector();

    // Check if two map are of same key and value type
    if (!ObjectInspectorUtils.compareTypes(leftMapKeyOI, rightMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + rightMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, leftMapKeyOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity key, but "
                        + "\"" + leftMapKeyOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(leftMapValueOI, rightMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + leftMapValueOI.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + rightMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, leftMapValueOI)) {
        throw new UDFArgumentTypeException(1,
                "\"" + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector.getTypeName() + "\""
                        + " expected at function cosine_similarity value, but "
                        + "\"" + leftMapValueOI.getTypeName() + "\""
                        + " is found");
    }

    return ObjectInspectorFactory.getStandardMapObjectInspector(leftMapKeyOI, leftMapValueOI);
}
 
Example 17
Source Project: incubator-hivemall   File: AUCUDAF.java    License: Apache License 2.0 4 votes vote down vote up
private static StructObjectInspector internalMergeOI() {
    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("indexScore");
    fieldOIs.add(writableDoubleObjectInspector);
    fieldNames.add("area");
    fieldOIs.add(writableDoubleObjectInspector);
    fieldNames.add("fp");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("tp");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("fpPrev");
    fieldOIs.add(writableLongObjectInspector);
    fieldNames.add("tpPrev");
    fieldOIs.add(writableLongObjectInspector);

    MapObjectInspector areaPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaDoubleObjectInspector);
    fieldNames.add("areaPartialMap");
    fieldOIs.add(areaPartialMapOI);

    MapObjectInspector fpPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("fpPartialMap");
    fieldOIs.add(fpPartialMapOI);

    MapObjectInspector tpPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("tpPartialMap");
    fieldOIs.add(tpPartialMapOI);

    MapObjectInspector fpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("fpPrevPartialMap");
    fieldOIs.add(fpPrevPartialMapOI);

    MapObjectInspector tpPrevPartialMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);
    fieldNames.add("tpPrevPartialMap");
    fieldOIs.add(tpPrevPartialMapOI);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 18
Source Project: incubator-hivemall   File: AUCUDAF.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void merge(AggregationBuffer agg, Object partial) throws HiveException {
    if (partial == null) {
        return;
    }

    Object indexScoreObj = internalMergeOI.getStructFieldData(partial, indexScoreField);
    Object areaObj = internalMergeOI.getStructFieldData(partial, areaField);
    Object fpObj = internalMergeOI.getStructFieldData(partial, fpField);
    Object tpObj = internalMergeOI.getStructFieldData(partial, tpField);
    Object fpPrevObj = internalMergeOI.getStructFieldData(partial, fpPrevField);
    Object tpPrevObj = internalMergeOI.getStructFieldData(partial, tpPrevField);
    Object areaPartialMapObj =
            internalMergeOI.getStructFieldData(partial, areaPartialMapField);
    Object fpPartialMapObj = internalMergeOI.getStructFieldData(partial, fpPartialMapField);
    Object tpPartialMapObj = internalMergeOI.getStructFieldData(partial, tpPartialMapField);
    Object fpPrevPartialMapObj =
            internalMergeOI.getStructFieldData(partial, fpPrevPartialMapField);
    Object tpPrevPartialMapObj =
            internalMergeOI.getStructFieldData(partial, tpPrevPartialMapField);

    double indexScore = writableDoubleObjectInspector.get(indexScoreObj);
    double area = writableDoubleObjectInspector.get(areaObj);
    long fp = writableLongObjectInspector.get(fpObj);
    long tp = writableLongObjectInspector.get(tpObj);
    long fpPrev = writableLongObjectInspector.get(fpPrevObj);
    long tpPrev = writableLongObjectInspector.get(tpPrevObj);

    StandardMapObjectInspector ddMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaDoubleObjectInspector);
    StandardMapObjectInspector dlMapOI =
            ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector,
                javaLongObjectInspector);

    Map<Double, Double> areaPartialMap = (Map<Double, Double>) ddMapOI.getMap(
        HiveUtils.castLazyBinaryObject(areaPartialMapObj));
    Map<Double, Long> fpPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(fpPartialMapObj));
    Map<Double, Long> tpPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(tpPartialMapObj));
    Map<Double, Long> fpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(fpPrevPartialMapObj));
    Map<Double, Long> tpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap(
        HiveUtils.castLazyBinaryObject(tpPrevPartialMapObj));

    ClassificationAUCAggregationBuffer myAggr = (ClassificationAUCAggregationBuffer) agg;
    myAggr.merge(indexScore, area, fp, tp, fpPrev, tpPrev, areaPartialMap, fpPartialMap,
        tpPartialMap, fpPrevPartialMap, tpPrevPartialMap);
}
 
Example 19
Source Project: incubator-hivemall   File: SlimUDTFTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAllSamples() throws HiveException {
    SlimUDTF slim = new SlimUDTF();
    ObjectInspector itemIOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector itemJOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;

    ObjectInspector itemIRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
    ObjectInspector itemJRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        PrimitiveObjectInspectorFactory.javaFloatObjectInspector);
    ObjectInspector topKRatesOfIOI = ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector,
        ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaIntObjectInspector,
            PrimitiveObjectInspectorFactory.javaFloatObjectInspector));
    ObjectInspector optionArgumentOI = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-l2 0.01 -l1 0.01");

    ObjectInspector[] argOIs =
            {itemIOI, itemIRatesOI, topKRatesOfIOI, itemJOI, itemJRatesOI, optionArgumentOI};

    slim.initialize(argOIs);
    int numUser = 4;
    int numItem = 5;

    float[][] data = {{1.f, 4.f, 0.f, 0.f, 0.f}, {0.f, 3.f, 0.f, 1.f, 2.f},
            {2.f, 2.f, 0.f, 0.f, 3.f}, {0.f, 1.f, 1.f, 0.f, 0.f}};

    for (int i = 0; i < numItem; i++) {
        Map<Integer, Float> Ri = new HashMap<>();
        for (int u = 0; u < numUser; u++) {
            if (data[u][i] != 0.) {
                Ri.put(u, data[u][i]);
            }
        }

        // most similar data
        Map<Integer, Map<Integer, Float>> knnRatesOfI = new HashMap<>();
        for (int u = 0; u < numUser; u++) {
            Map<Integer, Float> Ru = new HashMap<>();
            for (int k = 0; k < numItem; k++) {
                if (k == i)
                    continue;
                Ru.put(k, data[u][k]);
            }
            knnRatesOfI.put(u, Ru);
        }

        for (int j = 0; j < numItem; j++) {
            if (i == j)
                continue;
            Map<Integer, Float> Rj = new HashMap<>();
            for (int u = 0; u < numUser; u++) {
                if (data[u][j] != 0.) {
                    Rj.put(u, data[u][j]);
                }
            }

            Object[] args = {i, Ri, knnRatesOfI, j, Rj};
            slim.process(args);
        }
    }
    slim.finalizeTraining();
}
 
Example 20
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        processOptions(parameters);
        this.wordOI = HiveUtils.asStringOI(parameters[0]);
        this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);
        this.labelOI = HiveUtils.asIntegerOI(parameters[2]);
        this.probOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.wcListField = soi.getStructFieldRef("wcList");
        this.probMapField = soi.getStructFieldRef("probMap");
        this.topicsOptionField = soi.getStructFieldRef("topics");
        this.alphaOptionField = soi.getStructFieldRef("alpha");
        this.deltaOptionField = soi.getStructFieldRef("delta");
        this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);
        this.probMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.probMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        this.probMapValueOI =
                ObjectInspectorFactory.getStandardListObjectInspector(probMapValueElemOI);
        this.probMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(probMapKeyOI,
            probMapValueOI);
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {
        final ArrayList<String> fieldNames = new ArrayList<String>();
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldNames.add("label");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
        fieldNames.add("probability");
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);

        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs));
    }
    return outputOI;
}