Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTailK() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k -2")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 2
Source Project: incubator-hivemall   Source File: LDAUDTFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRow() throws HiveException {
    LDAUDTF udtf = new LDAUDTF();
    final int numTopics = 2;
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-topics " + numTopics)};
    udtf.initialize(argOIs);

    String[] doc1 = new String[] {"1", "2", "3"};
    udtf.process(new Object[] {Arrays.asList(doc1)});

    final MutableInt cnt = new MutableInt(0);
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object arg0) throws HiveException {
            cnt.addValue(1);
        }
    });
    udtf.close();

    Assert.assertEquals(doc1.length * numTopics, cnt.getValue());
}
 
Example 3
Source Project: hive-third-functions   Source File: UDFArrayValueCount.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    result.set(0L);

    Object array = arguments[ARRAY_IDX].get();
    Object value = arguments[VALUE_IDX].get();

    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty or value is null
    if (array == null || arrayLength <= 0) {
        return result;
    }

    long count = 0L;
    for (int i = 0; i < arrayLength; ++i) {
        Object listElement = arrayOI.getListElement(array, i);
        if (ObjectInspectorUtils.compare(value, valueOI, listElement, arrayElementOI) == 0) {
            count++;
        }
    }
    result.set(count);

    return result;
}
 
Example 4
Source Project: incubator-hivemall   Source File: ToLibSVMFormatUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFeatureOnly() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 10")});

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(new DeferredObject[] {
            new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1"))}));

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(
        new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1"))}));

    udf.close();
}
 
Example 5
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_multimap(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_multimap, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 
Example 6
Source Project: incubator-hivemall   Source File: MapRouletteUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example 7
Source Project: hive-third-functions   Source File: UDFRe2JRegexpSplit.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function regexp_split(string, pattern) takes exactly " + ARG_COUNT + " arguments.");
    }

    for (int i = 0; i < ARG_COUNT; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function regexp_split, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector expectOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;

    return ObjectInspectorFactory.getStandardListObjectInspector(expectOI);
}
 
Example 8
Source Project: incubator-hivemall   Source File: UDTFWithOptions.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
protected final List<FeatureValue> parseFeatures(@Nonnull final List<?> features,
        @Nonnull final ObjectInspector featureInspector, final boolean parseFeature) {
    final int numFeatures = features.size();
    if (numFeatures == 0) {
        return Collections.emptyList();
    }
    final List<FeatureValue> list = new ArrayList<FeatureValue>(numFeatures);
    for (Object f : features) {
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object o = ObjectInspectorUtils.copyToStandardObject(f, featureInspector,
                ObjectInspectorCopyOption.WRITABLE);
            Writable k = WritableUtils.toWritable(o);
            fv = new FeatureValue(k, 1.f);
        }
        list.add(fv);
    }
    return list;
}
 
Example 9
Source Project: incubator-hivemall   Source File: MapIncludeKeysUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException(
            "Expected two arguments for map_filter_keys: " + argOIs.length);
    }

    this.mapOI = HiveUtils.asMapOI(argOIs[0]);
    this.listOI = HiveUtils.asListOI(argOIs[1]);

    ObjectInspector mapKeyOI = mapOI.getMapKeyObjectInspector();
    ObjectInspector filterKeyOI = listOI.getListElementObjectInspector();
    if (!ObjectInspectorUtils.compareTypes(mapKeyOI, filterKeyOI)) {
        throw new UDFArgumentException("Element types does not match: mapKey "
                + mapKeyOI.getTypeName() + ", filterKey" + filterKeyOI.getTypeName());
    }

    return ObjectInspectorUtils.getStandardObjectInspector(mapOI,
        ObjectInspectorCopyOption.WRITABLE);
}
 
Example 10
@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 
Example 11
Source Project: incubator-hivemall   Source File: GuessAttributesUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    final StringBuilder buf = new StringBuilder(128);
    final int numArgs = argOIs.length;
    final int last = numArgs - 1;
    for (int i = 0; i < numArgs; i++) {
        if (HiveUtils.isNumberOI(argOIs[i])) {
            buf.append('Q'); // quantitative
        } else {
            buf.append('C'); // categorical            
        }
        if (i != last) {
            buf.append(',');
        }
    }
    String value = buf.toString();
    return ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, value);
}
 
Example 12
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReverseTopK() throws Exception {
    // = tail-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 13
Source Project: incubator-hivemall   Source File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 14
Source Project: incubator-hivemall   Source File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector terminalOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) {
    Preconditions.checkNotNull(inputOIs);
    Preconditions.checkArgument(inputOIs.length >= 1, inputOIs.length);

    final List<String> fieldNames = new ArrayList<>(inputOIs.length);
    final List<ObjectInspector> fieldOIs = new ArrayList<>(inputOIs.length);
    for (int i = 0; i < inputOIs.length; i++) {
        fieldNames.add("f" + String.valueOf(i + 1));
        ObjectInspector keyOI = ObjectInspectorUtils.getStandardObjectInspector(inputOIs[i],
            ObjectInspectorCopyOption.WRITABLE);
        MapObjectInspector mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            keyOI, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        fieldOIs.add(mapOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 15
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTop2IntNuturalOrder() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final Integer[] values = new Integer[] {3, -1, 4, 4, 2, 5};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals(5, res.get(0));
    Assert.assertEquals(4, res.get(1));
}
 
Example 16
Source Project: incubator-hivemall   Source File: ArrayFlattenUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 1) {
        throw new UDFArgumentException(
            "array_flatten expects exactly one argument: " + argOIs.length);
    }

    this.listOI = HiveUtils.asListOI(argOIs[0]);
    ObjectInspector listElemOI = listOI.getListElementObjectInspector();
    if (listElemOI.getCategory() != Category.LIST) {
        throw new UDFArgumentException(
            "array_flatten takes array of array for the argument: " + listOI.toString());
    }

    ListObjectInspector nestedListOI = HiveUtils.asListOI(listElemOI);
    ObjectInspector elemOI = nestedListOI.getListElementObjectInspector();

    return ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(elemOI,
            ObjectInspectorCopyOption.WRITABLE));
}
 
Example 17
Source Project: incubator-hivemall   Source File: UDAFToOrderedMap.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOI(
        @Nonnull PrimitiveObjectInspector keyOI, @Nonnull ObjectInspector valueOI) {
    List<String> fieldNames = new ArrayList<String>();
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("partialMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(keyOI),
        ObjectInspectorUtils.getStandardObjectInspector(valueOI)));

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 18
Source Project: incubator-hivemall   Source File: RegressionBaseUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
protected final FeatureValue[] parseFeatures(@Nonnull final List<?> features) {
    final int size = features.size();
    if (size == 0) {
        return null;
    }

    final ObjectInspector featureInspector = featureListOI.getListElementObjectInspector();
    final FeatureValue[] featureVector = new FeatureValue[size];
    for (int i = 0; i < size; i++) {
        Object f = features.get(i);
        if (f == null) {
            continue;
        }
        final FeatureValue fv;
        if (parseFeature) {
            fv = FeatureValue.parse(f);
        } else {
            Object k = ObjectInspectorUtils.copyToStandardObject(f, featureInspector);
            fv = new FeatureValue(k, 1.f);
        }
        featureVector[i] = fv;
    }
    return featureVector;
}
 
Example 19
Source Project: presto   Source File: TestHiveBucketing.java    License: Apache License 2.0 5 votes vote down vote up
public static int getHiveBucketHashCode(BucketingVersion bucketingVersion, List<Entry<ObjectInspector, Object>> columnBindings)
{
    ObjectInspector[] objectInspectors = new ObjectInspector[columnBindings.size()];
    Object[] objects = new Object[columnBindings.size()];

    int i = 0;
    for (Entry<ObjectInspector, Object> entry : columnBindings) {
        objectInspectors[i] = entry.getKey();
        if (entry.getValue() != null && entry.getKey() instanceof JavaHiveVarcharObjectInspector) {
            JavaHiveVarcharObjectInspector varcharObjectInspector = (JavaHiveVarcharObjectInspector) entry.getKey();
            objects[i] = new HiveVarchar(((String) entry.getValue()), varcharObjectInspector.getMaxLength());
        }
        else {
            objects[i] = entry.getValue();
        }
        i++;
    }

    switch (bucketingVersion) {
        case BUCKETING_V1:
            @SuppressWarnings("deprecation")
            int hashCodeOld = ObjectInspectorUtils.getBucketHashCodeOld(objects, objectInspectors);
            return hashCodeOld;
        case BUCKETING_V2:
            return ObjectInspectorUtils.getBucketHashCode(objects, objectInspectors);
        default:
            throw new IllegalArgumentException("Unsupported bucketing version: " + bucketingVersion);
    }
}
 
Example 20
Source Project: occurrence   Source File: ArrayNullsRemoverGenericUDF.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Removes empty Strings and null values.
 */
private List handlerStringArray(List list) {
  List result = Lists.newArrayList();
  for (Object oElement : list) {
    Object stdObject = ObjectInspectorUtils.copyToStandardJavaObject(oElement, primitiveObjectInspector);
    if (stdObject != null && !((String)stdObject).trim().isEmpty()) {
      result.add(stdObject);
    }
  }
  return result;
}
 
Example 21
/**
 * Perform argument count check and argument type checking, returns an
 * appropriate evaluator to perform based on input type (which should always
 * be BINARY sketch). Also check lgK and target HLL type parameters if they are passed in.
 *
 * @see org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver
 * #getEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo)
 *
 * @param info The parameter info to validate
 * @return The GenericUDAFEvaluator to use to compute the function.
 */
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info) throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();

  if (inspectors.length < 1) {
    throw new UDFArgumentException("Please specify at least 1 argument");
  }

  if (inspectors.length > 3) {
    throw new UDFArgumentTypeException(inspectors.length - 1, "Please specify no more than 3 arguments");
  }

  ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[0], 0, PrimitiveCategory.BINARY);

  // Validate second argument if present
  if (inspectors.length > 1) {
    ObjectInspectorValidator.validateIntegralParameter(inspectors[1], 1);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[1])) {
      throw new UDFArgumentTypeException(1, "The second argument must be a constant");
    }
  }

  // Validate third argument if present
  if (inspectors.length > 2) {
    ObjectInspectorValidator.validateGivenPrimitiveCategory(inspectors[2], 2, PrimitiveCategory.STRING);
    if (!ObjectInspectorUtils.isConstantObjectInspector(inspectors[2])) {
      throw new UDFArgumentTypeException(2, "The third argument must be a constant");
    }
  }

  return new UnionSketchUDAFEvaluator();
}
 
Example 22
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testKVMapTop2() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -kv_map")};

    final int[] keys = new int[] {5, 3, 4, 2, 3};
    final String[] values = new String[] {"apple", "banana", "candy", "donut", "egg"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(2, map.size());

    Assert.assertEquals("apple", map.get(5));
    Assert.assertEquals("candy", map.get(4));
}
 
Example 23
Source Project: hive-third-functions   Source File: UDFArrayMax.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function array_max(array) takes exactly " + ARG_COUNT + "arguments.");
    }

    // Check if two argument is of category LIST
    if (!arguments[0].getCategory().equals(ObjectInspector.Category.LIST)) {
        throw new UDFArgumentTypeException(0,
                "\"" + org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "\" "
                        + "expected at function array_max, but "
                        + "\"" + arguments[0].getTypeName() + "\" "
                        + "is found");
    }

    arrayOI = (ListObjectInspector) arguments[0];
    arrayElementOI = arrayOI.getListElementObjectInspector();

    // Check if the comparison is supported for this type
    if (!ObjectInspectorUtils.compareSupported(arrayElementOI)) {
        throw new UDFArgumentException("The function array_max"
                + " does not support comparison for "
                + "\"" + arrayElementOI.getTypeName() + "\""
                + " types");
    }

    return arrayElementOI;
}
 
Example 24
Source Project: hive-third-functions   Source File: UDFArrayContains.java    License: Apache License 2.0 5 votes vote down vote up
@Override
// Returns <tt>true</tt> if this array contains the specified element.
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    result.set(false);

    Object array = arguments[ARRAY_IDX].get();
    Object value = arguments[VALUE_IDX].get();

    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty or value is null
    if (array == null) {
        return result;
    }

    if (value == null || arrayLength <= 0) {
        return result;
    }

    // Compare the value to each element of array until a match is found
    for (int i = 0; i < arrayLength; ++i) {
        Object listElement = arrayOI.getListElement(array, i);
        if (listElement != null) {
            if (ObjectInspectorUtils.compare(value, valueOI, listElement, arrayElementOI) == 0) {
                result.set(true);
                break;
            }
        }
    }

    return result;
}
 
Example 25
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReverseTopKWithKey() throws Exception {
    // = tail-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 26
@Test
public void testRMSpropGraves() throws IOException, HiveException {
    String filePath = "adam_test_10000.tsv.gz";
    String options =
            "-loss logloss -opt RMSpropGraves -reg l1 -lambda 0.0001 -iter 10 -mini_batch 1 -cv_rate 0.00005";

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();

    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector, params});

    BufferedReader reader = readFile(filePath);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");

        String featureLine = tokenizer.nextToken();
        List<String> X = Arrays.asList(featureLine.split(","));

        String labelLine = tokenizer.nextToken();
        Integer y = Integer.valueOf(labelLine);

        udtf.process(new Object[] {X, y});
    }

    udtf.finalizeTraining();

    Assert.assertTrue(
        "CumulativeLoss is expected to be less than 1200: " + udtf.getCumulativeLoss(),
        udtf.getCumulativeLoss() < 1200);
}
 
Example 27
Source Project: incubator-hivemall   Source File: UDAFToOrderedMap.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        this.inputValueOI = argOIs[1];
        this.sizeOI = HiveUtils.asIntegerOI(argOIs[2]);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) argOIs[0];
        this.internalMergeOI = soi;

        this.partialMapField = soi.getStructFieldRef("partialMap");
        // re-extract input key/value OIs
        MapObjectInspector partialMapOI =
                (MapObjectInspector) partialMapField.getFieldObjectInspector();
        this.inputKeyOI = HiveUtils.asPrimitiveObjectInspector(
            partialMapOI.getMapKeyObjectInspector());
        this.inputValueOI = partialMapOI.getMapValueObjectInspector();

        this.partialMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));

        this.sizeField = soi.getStructFieldRef("size");
        this.sizeOI = (PrimitiveObjectInspector) sizeField.getFieldObjectInspector();
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI(inputKeyOI, inputValueOI);
    } else {// terminate
        outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputKeyOI),
            ObjectInspectorUtils.getStandardObjectInspector(inputValueOI));
    }
    return outputOI;
}
 
Example 28
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReverseTailKWithKey() throws Exception {
    // = top-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k -2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 29
Source Project: incubator-hivemall   Source File: LearnerBaseUDTF.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
protected ObjectInspector getFeatureOutputOI(@Nonnull PrimitiveObjectInspector featureInputOI)
        throws UDFArgumentException {
    if (dense_model) {
        // TODO validation
        return PrimitiveObjectInspectorFactory.javaIntObjectInspector; // see DenseModel
    }
    return ObjectInspectorUtils.getStandardObjectInspector(featureInputOI);
}
 
Example 30
Source Project: incubator-hivemall   Source File: MapRouletteUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 1 && argOIs.length != 2) {
        throw new UDFArgumentLengthException(
            "Expected exactly one argument for map_roulette: " + argOIs.length);
    }
    if (argOIs[0].getCategory() != ObjectInspector.Category.MAP) {
        throw new UDFArgumentTypeException(0,
            "Only map type argument is accepted but got " + argOIs[0].getTypeName());
    }

    this.mapOI = HiveUtils.asMapOI(argOIs[0]);
    this.valueOI = HiveUtils.asDoubleCompatibleOI(mapOI.getMapValueObjectInspector());

    if (argOIs.length == 2) {
        ObjectInspector argOI1 = argOIs[1];
        if (HiveUtils.isIntegerOI(argOI1) == false) {
            throw new UDFArgumentException(
                "The second argument of map_roulette must be integer type: "
                        + argOI1.getTypeName());
        }
        if (ObjectInspectorUtils.isConstantObjectInspector(argOI1)) {
            long seed = HiveUtils.getAsConstLong(argOI1);
            this._rand = new Random(seed); // fixed seed
        } else {
            this.seedOI = HiveUtils.asLongCompatibleOI(argOI1);
        }
    } else {
        this._rand = new Random(); // random seed
    }

    return mapOI.getMapKeyObjectInspector();
}