org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: UDTFWithOptions.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
protected final List<FeatureValue> parseFeatures(@Nonnull final List<?> features,
        @Nonnull final ObjectInspector featureInspector, final boolean parseFeature) {
    final int numFeatures = features.size();
    if (numFeatures == 0) {
        return Collections.emptyList();
    }
    final List<FeatureValue> list = new ArrayList<FeatureValue>(numFeatures);
    for (Object f : features) {
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object o = ObjectInspectorUtils.copyToStandardObject(f, featureInspector,
                ObjectInspectorCopyOption.WRITABLE);
            Writable k = WritableUtils.toWritable(o);
            fv = new FeatureValue(k, 1.f);
        }
        list.add(fv);
    }
    return list;
}
 
Example #2
Source File: LDAUDTFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRow() throws HiveException {
    LDAUDTF udtf = new LDAUDTF();
    final int numTopics = 2;
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-topics " + numTopics)};
    udtf.initialize(argOIs);

    String[] doc1 = new String[] {"1", "2", "3"};
    udtf.process(new Object[] {Arrays.asList(doc1)});

    final MutableInt cnt = new MutableInt(0);
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object arg0) throws HiveException {
            cnt.addValue(1);
        }
    });
    udtf.close();

    Assert.assertEquals(doc1.length * numTopics, cnt.getValue());
}
 
Example #3
Source File: RegressionBaseUDTF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 
Example #4
Source File: UDAFToOrderedMap.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOI(
        @Nonnull PrimitiveObjectInspector keyOI, @Nonnull ObjectInspector valueOI) {
    List<String> fieldNames = new ArrayList<String>();
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("partialMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(keyOI),
        ObjectInspectorUtils.getStandardObjectInspector(valueOI)));

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #5
Source File: ArrayFlattenUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 1) {
        throw new UDFArgumentException(
            "array_flatten expects exactly one argument: " + argOIs.length);
    }

    this.listOI = HiveUtils.asListOI(argOIs[0]);
    ObjectInspector listElemOI = listOI.getListElementObjectInspector();
    if (listElemOI.getCategory() != Category.LIST) {
        throw new UDFArgumentException(
            "array_flatten takes array of array for the argument: " + listOI.toString());
    }

    ListObjectInspector nestedListOI = HiveUtils.asListOI(listElemOI);
    ObjectInspector elemOI = nestedListOI.getListElementObjectInspector();

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(elemOI,
            ObjectInspectorCopyOption.WRITABLE));
}
 
Example #6
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTop2IntNuturalOrder() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final Integer[] values = new Integer[] {3, -1, 4, 4, 2, 5};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals(5, res.get(0));
    Assert.assertEquals(4, res.get(1));
}
 
Example #7
Source File: UDFArrayValueCount.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    result.set(0L);

    Object array = arguments[ARRAY_IDX].get();
    Object value = arguments[VALUE_IDX].get();

    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty or value is null
    if (array == null || arrayLength <= 0) {
        return result;
    }

    long count = 0L;
    for (int i = 0; i < arrayLength; ++i) {
        Object listElement = arrayOI.getListElement(array, i);
        if (ObjectInspectorUtils.compare(value, valueOI, listElement, arrayElementOI) == 0) {
            count++;
        }
    }
    result.set(count);

    return result;
}
 
Example #8
Source File: ToLibSVMFormatUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFeatureOnly() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 10")});

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(new DeferredObject[] {
            new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1"))}));

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(
        new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1"))}));

    udf.close();
}
 
Example #9
Source File: UDFStringSplitToMultimap.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_multimap, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 
Example #10
Source File: OnehotEncodingUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #11
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTailK() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k -2")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example #12
Source File: MapIncludeKeysUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException(
            "Expected two arguments for map_filter_keys: " + argOIs.length);
    }

    this.mapOI = HiveUtils.asMapOI(argOIs[0]);
    this.listOI = HiveUtils.asListOI(argOIs[1]);

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector filterKeyOI = listOI.getListElementObjectInspector();
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, filterKeyOI)) {
        throw new UDFArgumentException("Element types does not match: mapKey "
                + mapKeyOI.getTypeName() + ", filterKey" + filterKeyOI.getTypeName());
    }

    return ObjectInspectorUtils.getStandardObjectInspector(mapOI,
        ObjectInspectorCopyOption.WRITABLE);
}
 
Example #13
Source File: MulticlassOnlineClassifierUDTF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 
Example #14
Source File: UDFRe2JRegexpSplit.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function regexp_split(string, pattern) takes exactly " + ARG_COUNT + " arguments.");
    }

    for (int i = 0; i < ARG_COUNT; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function regexp_split, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector expectOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;

    return ObjectInspectorFactory.getStandardListObjectInspector(expectOI);
}
 
Example #15
Source File: GuessAttributesUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final StringBuilder buf = new StringBuilder(128);
    final int numArgs = argOIs.length;
    final int last = numArgs - 1;
    for (int i = 0; i < numArgs; i++) {
        if (HiveUtils.isNumberOI(argOIs[i])) {
            buf.append('Q'); // quantitative
        } else {
            buf.append('C'); // categorical            
        }
        if (i != last) {
            buf.append(',');
        }
    }
    String value = buf.toString();
    return ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, value);
}
 
Example #16
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example #17
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testReverseTopK() throws Exception {
    // = tail-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example #18
Source File: OnehotEncodingUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #19
Source File: GeneralClassifierUDTFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testSGD() throws IOException, HiveException {
    String filePath = "adam_test_10000.tsv.gz";
    String options =
            "-loss logloss -opt sgd -reg l1 -lambda 0.0001 -iter 10 -mini_batch 1 -cv_rate 0.00005";

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();

    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector, params});

    BufferedReader reader = readFile(filePath);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");

        String featureLine = tokenizer.nextToken();
        List<String> X = Arrays.asList(featureLine.split(","));

        String labelLine = tokenizer.nextToken();
        Integer y = Integer.valueOf(labelLine);

        udtf.process(new Object[] {X, y});
    }

    udtf.finalizeTraining();

    Assert.assertTrue(
        "CumulativeLoss is expected to be less than 1300: " + udtf.getCumulativeLoss(),
        udtf.getCumulativeLoss() < 1300);
}
 
Example #20
Source File: TestHiveBucketing.java    From presto with Apache License 2.0 5 votes vote down vote up
public static int getHiveBucketHashCode(BucketingVersion bucketingVersion, List<Entry<ObjectInspector, Object>> columnBindings)
{
    ObjectInspector[] objectInspectors = new ObjectInspector[columnBindings.size()];
    Object[] objects = new Object[columnBindings.size()];

    int i = 0;
    for (Entry<ObjectInspector, Object> entry : columnBindings) {
        objectInspectors[i] = entry.getKey();
        if (entry.getValue() != null && entry.getKey() instanceof JavaHiveVarcharObjectInspector) {
            JavaHiveVarcharObjectInspector varcharObjectInspector = (JavaHiveVarcharObjectInspector) entry.getKey();
            objects[i] = new HiveVarchar(((String) entry.getValue()), varcharObjectInspector.getMaxLength());
        }
        else {
            objects[i] = entry.getValue();
        }
        i++;
    }

    switch (bucketingVersion) {
        case BUCKETING_V1:
            @SuppressWarnings("deprecation")
            int hashCodeOld = ObjectInspectorUtils.getBucketHashCodeOld(objects, objectInspectors);
            return hashCodeOld;
        case BUCKETING_V2:
            return ObjectInspectorUtils.getBucketHashCode(objects, objectInspectors);
        default:
            throw new IllegalArgumentException("Unsupported bucketing version: " + bucketingVersion);
    }
}
 
Example #21
Source File: FMeasureUDAFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
private void setUpWithArguments(double beta, String average) throws Exception {
    fmeasure = new FMeasureUDAF();
    inputOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-beta " + beta + " -average " + average)};

    evaluator =
            fmeasure.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));
    agg = (FMeasureUDAF.FMeasureAggregationBuffer) evaluator.getNewAggregationBuffer();
}
 
Example #22
Source File: UnionSketchUDAF.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
/**
 * Perform argument count check and argument type checking, returns an
 * appropriate evaluator to perform based on input type (which should always
 * be BINARY sketch). Also check lgK and seed parameters if they are passed in.
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info The parameter info to validate
 * @return The GenericUDAFEvaluator to use to compute the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (inspectors.length > 3) {
    throw new UDFArgumentTypeException(inspectors.length - 1, "Please specify no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // Validate second argument if present
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[1])) {
      throw new UDFArgumentTypeException(1, "The second argument must be a constant");
    }
  }

  // Validate third argument if present
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[2], 2);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[2])) {
      throw new UDFArgumentTypeException(2, "The third argument must be a constant");
    }
  }

  return new UnionSketchUDAFEvaluator();
}
 
Example #23
Source File: MaxRowUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    final MaxAgg maxagg = (MaxAgg) agg;

    final List<Object> otherObjects;
    if (partial instanceof Object[]) {
        otherObjects = Arrays.asList((Object[]) partial);
    } else if (partial instanceof LazyBinaryStruct) {
        otherObjects = ((LazyBinaryStruct) partial).getFieldsAsList();
    } else if (inputStructOI != null) {
        otherObjects = inputStructOI.getStructFieldsDataAsList(partial);
    } else {
        throw new HiveException("Invalid type: " + partial.getClass().getName());
    }

    boolean isMax = false;
    if (maxagg.objects == null) {
        isMax = true;
    } else {
        int cmp = ObjectInspectorUtils.compare(maxagg.objects[0], outputOIs[0],
            otherObjects.get(0), inputOIs[0]);
        if (cmp < 0) {
            isMax = true;
        }
    }

    if (isMax) {
        int length = otherObjects.size();
        maxagg.objects = new Object[length];
        for (int i = 0; i < length; i++) {
            maxagg.objects[i] = ObjectInspectorUtils.copyToStandardObject(
                otherObjects.get(i), inputOIs[i]);
        }
    }
}
 
Example #24
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testVKMapOptionBananaOverlap2() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "banana", "candy"};
    final double[] keys = new double[] {0.8, 0.8, 0.7};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(1, map.size());

    Assert.assertEquals(0.8d, map.get("banana"));
}
 
Example #25
Source File: MinByUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
void merge(final Object newX, final Object newY,
        @Nonnull final ObjectInspector xInputOI,
        @Nonnull final ObjectInspector yInputOI,
        @Nonnull final ObjectInspector yOutputOI) {
    final int cmp = ObjectInspectorUtils.compare(y, yOutputOI, newY, yInputOI);
    if (x == null || cmp > 0) {// found smaller y
        this.x = ObjectInspectorUtils.copyToStandardObject(newX, xInputOI,
            ObjectInspectorCopyOption.JAVA);
        this.y = ObjectInspectorUtils.copyToStandardObject(newY, yInputOI,
            ObjectInspectorCopyOption.JAVA);
    }
}
 
Example #26
Source File: MatrixFactorizationAdaGradUDTFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() throws HiveException {
    TestUtils.testGenericUDTFSerialization(MatrixFactorizationAdaGradUDTF.class,
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-factor 10")},
        new Object[][] {{0, 0, 5.f}});
}
 
Example #27
Source File: GeneralClassifierUDTFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = UDFArgumentException.class)
public void testUnsupportedOptimizer() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-opt UnsupportedOpt");

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}
 
Example #28
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testVKMapOptionBananaOverlap() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "banana", "candy"};
    final double[] keys = new double[] {0.7, 0.8, 0.81};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(2, map.size());

    Assert.assertEquals(0.81d, map.get("candy"));
    Assert.assertEquals(0.8d, map.get("banana"));
}
 
Example #29
Source File: TryCastUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
    Object arg0 = args[0].get();
    if (arg0 == null) {
        return null;
    }

    Object input = ObjectInspectorUtils.copyToStandardObject(arg0, inputOI);
    try {
        return converter.convert(input);
    } catch (Exception e) {
        return null;
    }
}
 
Example #30
Source File: MatrixFactorizationSGDUDTFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultInit() throws HiveException {
    println("--------------------------\n testGaussian()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    //ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String("-factor 3 -eta 0.0002"));
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String("-factor 3"));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    mf.initialize(argOIs);
    Assert.assertTrue(mf.rankInit == RankInitScheme.random);

    float[][] rating = {{5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];
    final int num_iters = 100;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                //print(row + "," + col + ",");
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                //println((float) rating[row][col]);
                mf.process(args);
            }
        }
    }
    for (int row = 0; row < rating.length; row++) {
        for (int col = 0, size = rating[row].length; col < size; col++) {
            double predicted = mf.predict(row, col);
            print(rating[row][col] + "[" + predicted + "]\t");
            Assert.assertEquals(rating[row][col], predicted, 0.2d);
        }
        println();
    }
}