Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector()
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardMapObjectInspector() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: UDAFToMap.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Override public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException { super.init(mode, argOIs); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]); inputValueOI = argOIs[1]; } else {// from partial aggregation internalMergeOI = (StandardMapObjectInspector) argOIs[0]; inputKeyOI = HiveUtils.asPrimitiveObjectInspector( internalMergeOI.getMapKeyObjectInspector()); inputValueOI = internalMergeOI.getMapValueObjectInspector(); } return ObjectInspectorFactory.getStandardMapObjectInspector( ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI), ObjectInspectorUtils.getStandardObjectInspector(inputValueOI)); }
Example 2
Source File: MergeMapsUDAF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException { Preconditions.checkArgument(parameters.length == 1); super.init(mode, parameters); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data this.inputMapOI = HiveUtils.asMapOI(parameters[0]); this.inputKeyOI = inputMapOI.getMapKeyObjectInspector(); this.inputValOI = inputMapOI.getMapValueObjectInspector(); } else {// from partial aggregation this.mergeMapOI = HiveUtils.asMapOI(parameters[0]); this.inputKeyOI = mergeMapOI.getMapKeyObjectInspector(); this.inputValOI = mergeMapOI.getMapValueObjectInspector(); } return ObjectInspectorFactory.getStandardMapObjectInspector( ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI), ObjectInspectorUtils.getStandardObjectInspector(inputValOI)); }
Example 3
Source File: OnehotEncodingUDAF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull private static StructObjectInspector terminalOutputOI( @CheckForNull PrimitiveObjectInspector[] inputOIs) { Preconditions.checkNotNull(inputOIs); Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length); final List<String> fieldNames = new ArrayList<>(inputOIs.length); final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length); for (int i = 0; i < inputOIs.length; i++) { fieldNames.add("f" + String.valueOf(i + 1)); ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i], ObjectInspectorCopyOption.WRITABLE); MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector( keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector); fieldOIs.add(mapOI); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 4
Source File: UDFMapConcatTest.java From hive-third-functions with Apache License 2.0 | 6 votes |
@Test public void testMapConcat() throws Exception { UDFMapConcat udf = new UDFMapConcat(); ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector); ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector); ObjectInspector[] arguments = {leftMapOI, rightMapOI}; udf.initialize(arguments); LinkedHashMap<String, String> leftMap = Maps.newLinkedHashMap(); leftMap.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "13")); LinkedHashMap<String, String> rightMap = Maps.newLinkedHashMap(); rightMap.putAll(ImmutableMap.<String, String>of("key3", "21", "key4", "22", "key5", "23")); DeferredObject leftMapObj = new DeferredJavaObject(leftMap); DeferredObject rightMapObj = new DeferredJavaObject(rightMap); DeferredObject[] args = {leftMapObj, rightMapObj}; LinkedHashMap<String, String> output = (LinkedHashMap<String, String>) udf.evaluate(args); LinkedHashMap<String, String> expect = Maps.newLinkedHashMap(); expect.putAll(ImmutableMap.<String, String>of("key1", "11", "key2", "12", "key3", "21", "key4", "22", "key5", "23")); Assert.assertEquals("map_concat() test", true, MapUtils.mapEquals(output, expect)); }
Example 5
Source File: MapTailNUDF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length != 2) { throw new UDFArgumentLengthException( "map_tail_n only takes 2 arguments: map<object, object>, int"); } if (!(arguments[0] instanceof MapObjectInspector)) { throw new UDFArgumentException("The first argument must be a map"); } this.mapObjectInspector = (MapObjectInspector) arguments[0]; if (!(arguments[1] instanceof IntObjectInspector)) { throw new UDFArgumentException("The second argument must be an int"); } this.intObjectInspector = (IntObjectInspector) arguments[1]; ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector( mapObjectInspector.getMapKeyObjectInspector()); ObjectInspector valueOI = mapObjectInspector.getMapValueObjectInspector(); return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI); }
Example 6
Source File: UDFStringSplitToMultimap.java From hive-third-functions with Apache License 2.0 | 6 votes |
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { // Check if two arguments were passed if (arguments.length != ARG_COUNT) { throw new UDFArgumentLengthException( "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments."); } // Check if two argument is of string for (int i = 0; i < 3; i++) { if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) { throw new UDFArgumentTypeException(i, "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" " + "expected at function split_to_multimap, but " + "\"" + arguments[i].getTypeName() + "\" " + "is found"); } } ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector); return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI); }
Example 7
Source File: UDFMathCosineSimilarityTest.java From hive-third-functions with Apache License 2.0 | 5 votes |
public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException { UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity(); ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); ObjectInspector[] arguments = {leftMapOI, rightMapOI}; udf.initialize(arguments); GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap); GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap); GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj}; DoubleWritable output = (DoubleWritable) udf.evaluate(args); return output == null ? null : output.get(); }
Example 8
Source File: MajorityVoteUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public ObjectInspector init(@Nonnull Mode mode, @Nonnull ObjectInspector[] argOIs) throws HiveException { assert (argOIs.length == 1); super.init(mode, argOIs); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data this.keyInputOI = asPrimitiveObjectInspector(argOIs[0]); } else {// from partial aggregation this.partialOI = (StandardMapObjectInspector) argOIs[0]; this.keyInputOI = asPrimitiveObjectInspector(partialOI.getMapKeyObjectInspector()); this.counterInputOI = asLongOI(partialOI.getMapValueObjectInspector()); } this.keyOutputOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( keyInputOI.getTypeInfo()); // initialize output final ObjectInspector outputOI; if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOutputOI, PrimitiveObjectInspectorFactory.javaLongObjectInspector); } else {// terminate outputOI = keyOutputOI; } return outputOI; }
Example 9
Source File: HiveInspectors.java From flink with Apache License 2.0 | 5 votes |
private static ObjectInspector getObjectInspector(TypeInfo type) { switch (type.getCategory()) { case PRIMITIVE: PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type; return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType); case LIST: ListTypeInfo listType = (ListTypeInfo) type; return ObjectInspectorFactory.getStandardListObjectInspector( getObjectInspector(listType.getListElementTypeInfo())); case MAP: MapTypeInfo mapType = (MapTypeInfo) type; return ObjectInspectorFactory.getStandardMapObjectInspector( getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo())); case STRUCT: StructTypeInfo structType = (StructTypeInfo) type; List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos(); List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>(); for (TypeInfo fieldType : fieldTypes) { fieldInspectors.add(getObjectInspector(fieldType)); } return ObjectInspectorFactory.getStandardStructObjectInspector( structType.getAllStructFieldNames(), fieldInspectors); default: throw new CatalogException("Unsupported Hive type category " + type.getCategory()); } }
Example 10
Source File: HiveTypeSystem.java From transport with BSD 2-Clause "Simplified" License | 5 votes |
@Override protected ObjectInspector createMapType(ObjectInspector keyType, ObjectInspector valueType) { return ObjectInspectorFactory.getStandardMapObjectInspector( keyType, valueType ); }
Example 11
Source File: UDFMathCosineSimilarity.java From hive-third-functions with Apache License 2.0 | 4 votes |
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { // Check if two arguments were passed if (arguments.length != ARG_COUNT) { throw new UDFArgumentLengthException( "The function cosine_similarity(map, map) takes exactly " + ARG_COUNT + " arguments."); } // Check if two argument is of category LIST for (int i = 0; i < 2; i++) { if (!arguments[i].getCategory().equals(ObjectInspector.Category.MAP)) { throw new UDFArgumentTypeException(i, "\"" + serdeConstants.MAP_TYPE_NAME + "\" " + "expected at function cosine_similarity, but " + "\"" + arguments[i].getTypeName() + "\" " + "is found"); } } leftMapOI = (MapObjectInspector) arguments[0]; rightMapOI = (MapObjectInspector) arguments[1]; ObjectInspector leftMapKeyOI = leftMapOI.getMapKeyObjectInspector(); ObjectInspector leftMapValueOI = leftMapOI.getMapValueObjectInspector(); ObjectInspector rightMapKeyOI = rightMapOI.getMapKeyObjectInspector(); ObjectInspector rightMapValueOI = rightMapOI.getMapValueObjectInspector(); // Check if two map are of same key and value type if (!ObjectInspectorUtils.compareTypes(leftMapKeyOI, rightMapKeyOI)) { throw new UDFArgumentTypeException(1, "\"" + leftMapKeyOI.getTypeName() + "\"" + " expected at function cosine_similarity key, but " + "\"" + rightMapKeyOI.getTypeName() + "\"" + " is found"); } if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, leftMapKeyOI)) { throw new UDFArgumentTypeException(1, "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\"" + " expected at function cosine_similarity key, but " + "\"" + leftMapKeyOI.getTypeName() + "\"" + " is found"); } if (!ObjectInspectorUtils.compareTypes(leftMapValueOI, rightMapValueOI)) { throw new UDFArgumentTypeException(1, "\"" + leftMapValueOI.getTypeName() + "\"" + " expected at function cosine_similarity value, but " + "\"" + rightMapValueOI.getTypeName() + "\"" + " is found"); } if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, leftMapValueOI)) { throw new UDFArgumentTypeException(1, "\"" + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector.getTypeName() + "\"" + " expected at function cosine_similarity value, but " + "\"" + leftMapValueOI.getTypeName() + "\"" + " is found"); } return ObjectInspectorFactory.getStandardMapObjectInspector(leftMapKeyOI, leftMapValueOI); }
Example 12
Source File: RcFileTester.java From presto with Apache License 2.0 | 4 votes |
private static ObjectInspector getJavaObjectInspector(Type type) { if (type.equals(BOOLEAN)) { return javaBooleanObjectInspector; } if (type.equals(BIGINT)) { return javaLongObjectInspector; } if (type.equals(INTEGER)) { return javaIntObjectInspector; } if (type.equals(SMALLINT)) { return javaShortObjectInspector; } if (type.equals(TINYINT)) { return javaByteObjectInspector; } if (type.equals(REAL)) { return javaFloatObjectInspector; } if (type.equals(DOUBLE)) { return javaDoubleObjectInspector; } if (type instanceof VarcharType) { return javaStringObjectInspector; } if (type.equals(VARBINARY)) { return javaByteArrayObjectInspector; } if (type.equals(DATE)) { return javaDateObjectInspector; } if (type.equals(TIMESTAMP)) { return javaTimestampObjectInspector; } if (type instanceof DecimalType) { DecimalType decimalType = (DecimalType) type; return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale())); } if (type instanceof ArrayType) { return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0))); } if (type instanceof MapType) { ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0)); ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1)); return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); } if (type instanceof RowType) { return getStandardStructObjectInspector( type.getTypeSignature().getParameters().stream() .map(parameter -> parameter.getNamedTypeSignature().getName().get()) .collect(toList()), type.getTypeParameters().stream() .map(RcFileTester::getJavaObjectInspector) .collect(toList())); } throw new IllegalArgumentException("unsupported type: " + type); }
Example 13
Source File: StandardObjectInspectorFactory.java From transport with BSD 2-Clause "Simplified" License | 4 votes |
public static ObjectInspector map(ObjectInspector keyObjectInspector, ObjectInspector valueObjectInspector) { return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); }
Example 14
Source File: AUCUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
private static StructObjectInspector internalMergeOI() { ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("indexScore"); fieldOIs.add(writableDoubleObjectInspector); fieldNames.add("area"); fieldOIs.add(writableDoubleObjectInspector); fieldNames.add("fp"); fieldOIs.add(writableLongObjectInspector); fieldNames.add("tp"); fieldOIs.add(writableLongObjectInspector); fieldNames.add("fpPrev"); fieldOIs.add(writableLongObjectInspector); fieldNames.add("tpPrev"); fieldOIs.add(writableLongObjectInspector); MapObjectInspector areaPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaDoubleObjectInspector); fieldNames.add("areaPartialMap"); fieldOIs.add(areaPartialMapOI); MapObjectInspector fpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaLongObjectInspector); fieldNames.add("fpPartialMap"); fieldOIs.add(fpPartialMapOI); MapObjectInspector tpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaLongObjectInspector); fieldNames.add("tpPartialMap"); fieldOIs.add(tpPartialMapOI); MapObjectInspector fpPrevPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaLongObjectInspector); fieldNames.add("fpPrevPartialMap"); fieldOIs.add(fpPrevPartialMapOI); MapObjectInspector tpPrevPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaLongObjectInspector); fieldNames.add("tpPrevPartialMap"); fieldOIs.add(tpPrevPartialMapOI); return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
Example 15
Source File: AUCUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial == null) { return; } Object indexScoreObj = internalMergeOI.getStructFieldData(partial, indexScoreField); Object areaObj = internalMergeOI.getStructFieldData(partial, areaField); Object fpObj = internalMergeOI.getStructFieldData(partial, fpField); Object tpObj = internalMergeOI.getStructFieldData(partial, tpField); Object fpPrevObj = internalMergeOI.getStructFieldData(partial, fpPrevField); Object tpPrevObj = internalMergeOI.getStructFieldData(partial, tpPrevField); Object areaPartialMapObj = internalMergeOI.getStructFieldData(partial, areaPartialMapField); Object fpPartialMapObj = internalMergeOI.getStructFieldData(partial, fpPartialMapField); Object tpPartialMapObj = internalMergeOI.getStructFieldData(partial, tpPartialMapField); Object fpPrevPartialMapObj = internalMergeOI.getStructFieldData(partial, fpPrevPartialMapField); Object tpPrevPartialMapObj = internalMergeOI.getStructFieldData(partial, tpPrevPartialMapField); double indexScore = writableDoubleObjectInspector.get(indexScoreObj); double area = writableDoubleObjectInspector.get(areaObj); long fp = writableLongObjectInspector.get(fpObj); long tp = writableLongObjectInspector.get(tpObj); long fpPrev = writableLongObjectInspector.get(fpPrevObj); long tpPrev = writableLongObjectInspector.get(tpPrevObj); StandardMapObjectInspector ddMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaDoubleObjectInspector); StandardMapObjectInspector dlMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(javaDoubleObjectInspector, javaLongObjectInspector); Map<Double, Double> areaPartialMap = (Map<Double, Double>) ddMapOI.getMap( HiveUtils.castLazyBinaryObject(areaPartialMapObj)); Map<Double, Long> fpPartialMap = (Map<Double, Long>) dlMapOI.getMap( HiveUtils.castLazyBinaryObject(fpPartialMapObj)); Map<Double, Long> tpPartialMap = (Map<Double, Long>) dlMapOI.getMap( HiveUtils.castLazyBinaryObject(tpPartialMapObj)); Map<Double, Long> fpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap( HiveUtils.castLazyBinaryObject(fpPrevPartialMapObj)); Map<Double, Long> tpPrevPartialMap = (Map<Double, Long>) dlMapOI.getMap( HiveUtils.castLazyBinaryObject(tpPrevPartialMapObj)); ClassificationAUCAggregationBuffer myAggr = (ClassificationAUCAggregationBuffer) agg; myAggr.merge(indexScore, area, fp, tp, fpPrev, tpPrev, areaPartialMap, fpPartialMap, tpPartialMap, fpPrevPartialMap, tpPrevPartialMap); }
Example 16
Source File: SlimUDTFTest.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Test public void testAllSamples() throws HiveException { SlimUDTF slim = new SlimUDTF(); ObjectInspector itemIOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector; ObjectInspector itemJOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector; ObjectInspector itemIRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaFloatObjectInspector); ObjectInspector itemJRatesOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaFloatObjectInspector); ObjectInspector topKRatesOfIOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaFloatObjectInspector)); ObjectInspector optionArgumentOI = ObjectInspectorUtils.getConstantObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-l2 0.01 -l1 0.01"); ObjectInspector[] argOIs = {itemIOI, itemIRatesOI, topKRatesOfIOI, itemJOI, itemJRatesOI, optionArgumentOI}; slim.initialize(argOIs); int numUser = 4; int numItem = 5; float[][] data = {{1.f, 4.f, 0.f, 0.f, 0.f}, {0.f, 3.f, 0.f, 1.f, 2.f}, {2.f, 2.f, 0.f, 0.f, 3.f}, {0.f, 1.f, 1.f, 0.f, 0.f}}; for (int i = 0; i < numItem; i++) { Map<Integer, Float> Ri = new HashMap<>(); for (int u = 0; u < numUser; u++) { if (data[u][i] != 0.) { Ri.put(u, data[u][i]); } } // most similar data Map<Integer, Map<Integer, Float>> knnRatesOfI = new HashMap<>(); for (int u = 0; u < numUser; u++) { Map<Integer, Float> Ru = new HashMap<>(); for (int k = 0; k < numItem; k++) { if (k == i) continue; Ru.put(k, data[u][k]); } knnRatesOfI.put(u, Ru); } for (int j = 0; j < numItem; j++) { if (i == j) continue; Map<Integer, Float> Rj = new HashMap<>(); for (int u = 0; u < numUser; u++) { if (data[u][j] != 0.) { Rj.put(u, data[u][j]); } } Object[] args = {i, Ri, knnRatesOfI, j, Rj}; slim.process(args); } } slim.finalizeTraining(); }
Example 17
Source File: AUCUDAFTest.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Before public void setUp() throws Exception { auc = new AUCUDAF(); inputOIs = new ObjectInspector[] { PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( PrimitiveObjectInspector.PrimitiveCategory.DOUBLE), PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( PrimitiveObjectInspector.PrimitiveCategory.INT)}; evaluator = auc.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false)); ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("indexScore"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("area"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("fp"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("tp"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("fpPrev"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("tpPrev"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); MapObjectInspector areaPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableDoubleObjectInspector); fieldNames.add("areaPartialMap"); fieldOIs.add(areaPartialMapOI); MapObjectInspector fpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("fpPartialMap"); fieldOIs.add(fpPartialMapOI); MapObjectInspector tpPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("tpPartialMap"); fieldOIs.add(tpPartialMapOI); MapObjectInspector fpPrevPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("fpPrevPartialMap"); fieldOIs.add(fpPrevPartialMapOI); MapObjectInspector tpPrevPartialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector); fieldNames.add("tpPrevPartialMap"); fieldOIs.add(tpPrevPartialMapOI); partialOI = new ObjectInspector[2]; partialOI[0] = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); agg = (AUCUDAF.ClassificationAUCAggregationBuffer) evaluator.getNewAggregationBuffer(); }
Example 18
Source File: LDAPredictUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Override public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException { assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5); super.init(mode, parameters); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data processOptions(parameters); this.wordOI = HiveUtils.asStringOI(parameters[0]); this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]); this.labelOI = HiveUtils.asIntegerOI(parameters[2]); this.lambdaOI = HiveUtils.asDoubleCompatibleOI(parameters[3]); } else {// from partial aggregation StructObjectInspector soi = (StructObjectInspector) parameters[0]; this.internalMergeOI = soi; this.wcListField = soi.getStructFieldRef("wcList"); this.lambdaMapField = soi.getStructFieldRef("lambdaMap"); this.topicsOptionField = soi.getStructFieldRef("topics"); this.alphaOptionField = soi.getStructFieldRef("alpha"); this.deltaOptionField = soi.getStructFieldRef("delta"); this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI); this.lambdaMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.lambdaMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.lambdaMapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(lambdaMapValueElemOI); this.lambdaMapOI = ObjectInspectorFactory.getStandardMapObjectInspector( lambdaMapKeyOI, lambdaMapValueOI); } // initialize output final ObjectInspector outputOI; if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial outputOI = internalMergeOI(); } else { final ArrayList<String> fieldNames = new ArrayList<String>(); final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("label"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldNames.add("probability"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector); outputOI = ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs)); } return outputOI; }
Example 19
Source File: PLSAPredictUDAF.java From incubator-hivemall with Apache License 2.0 | 4 votes |
@Override public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException { assert (parameters.length == 1 || parameters.length == 4 || parameters.length == 5); super.init(mode, parameters); // initialize input if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data processOptions(parameters); this.wordOI = HiveUtils.asStringOI(parameters[0]); this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]); this.labelOI = HiveUtils.asIntegerOI(parameters[2]); this.probOI = HiveUtils.asDoubleCompatibleOI(parameters[3]); } else {// from partial aggregation StructObjectInspector soi = (StructObjectInspector) parameters[0]; this.internalMergeOI = soi; this.wcListField = soi.getStructFieldRef("wcList"); this.probMapField = soi.getStructFieldRef("probMap"); this.topicsOptionField = soi.getStructFieldRef("topics"); this.alphaOptionField = soi.getStructFieldRef("alpha"); this.deltaOptionField = soi.getStructFieldRef("delta"); this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI); this.probMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.probMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector; this.probMapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(probMapValueElemOI); this.probMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(probMapKeyOI, probMapValueOI); } // initialize output final ObjectInspector outputOI; if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial outputOI = internalMergeOI(); } else { final ArrayList<String> fieldNames = new ArrayList<String>(); final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); fieldNames.add("label"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); fieldNames.add("probability"); fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector); outputOI = ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs)); } return outputOI; }
Example 20
Source File: HiveWriteUtils.java From presto with Apache License 2.0 | 4 votes |
public static ObjectInspector getJavaObjectInspector(Type type) { if (type.equals(BooleanType.BOOLEAN)) { return javaBooleanObjectInspector; } if (type.equals(BigintType.BIGINT)) { return javaLongObjectInspector; } if (type.equals(IntegerType.INTEGER)) { return javaIntObjectInspector; } if (type.equals(SmallintType.SMALLINT)) { return javaShortObjectInspector; } if (type.equals(TinyintType.TINYINT)) { return javaByteObjectInspector; } if (type.equals(RealType.REAL)) { return javaFloatObjectInspector; } if (type.equals(DoubleType.DOUBLE)) { return javaDoubleObjectInspector; } if (type instanceof VarcharType) { return writableStringObjectInspector; } if (type instanceof CharType) { return writableHiveCharObjectInspector; } if (type.equals(VarbinaryType.VARBINARY)) { return javaByteArrayObjectInspector; } if (type.equals(DateType.DATE)) { return javaDateObjectInspector; } if (type.equals(TimestampType.TIMESTAMP)) { return javaTimestampObjectInspector; } if (type instanceof DecimalType) { DecimalType decimalType = (DecimalType) type; return getPrimitiveJavaObjectInspector(new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale())); } if (isArrayType(type)) { return ObjectInspectorFactory.getStandardListObjectInspector(getJavaObjectInspector(type.getTypeParameters().get(0))); } if (isMapType(type)) { ObjectInspector keyObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(0)); ObjectInspector valueObjectInspector = getJavaObjectInspector(type.getTypeParameters().get(1)); return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); } if (isRowType(type)) { return ObjectInspectorFactory.getStandardStructObjectInspector( type.getTypeSignature().getParameters().stream() .map(parameter -> parameter.getNamedTypeSignature().getName().get()) .collect(toImmutableList()), type.getTypeParameters().stream() .map(HiveWriteUtils::getJavaObjectInspector) .collect(toImmutableList())); } throw new IllegalArgumentException("unsupported type: " + type); }