org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RandomForestEnsembleUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
void merge(int size, @Nonnull Object posterioriObj,
        @Nonnull StandardListObjectInspector posterioriOI) throws HiveException {

    if (size != _k) {
        if (_k == -1) {
            this._k = size;
            this._posteriori = new double[size];
        } else {
            throw new HiveException(
                "Mismatch in the number of elements: _k=" + _k + ", size=" + size);
        }
    }

    final double[] posteriori = _posteriori;
    final DoubleObjectInspector doubleOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    for (int i = 0, len = _k; i < len; i++) {
        Object o2 = posterioriOI.getListElement(posterioriObj, i);
        posteriori[i] += doubleOI.get(o2);
    }
}
 
Example #2
Source File: CollectAllUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
    super.init(m, parameters);
    if (m == Mode.PARTIAL1) {
        inputOI = parameters[0];
        return ObjectInspectorFactory.getStandardListObjectInspector(
            ObjectInspectorUtils.getStandardObjectInspector(inputOI));
    } else {
        if (!(parameters[0] instanceof StandardListObjectInspector)) {
            inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
            return (StandardListObjectInspector) ObjectInspectorFactory.getStandardListObjectInspector(
                inputOI);
        } else {
            internalMergeOI = (StandardListObjectInspector) parameters[0];
            inputOI = internalMergeOI.getListElementObjectInspector();
            loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(
                internalMergeOI);
            return loi;
        }
    }
}
 
Example #3
Source File: ArrayAvgGenericUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
void merge(final int o_size, @Nonnull final Object o_sum, @Nonnull final Object o_count,
        @Nonnull final StandardListObjectInspector sumOI,
        @Nonnull final StandardListObjectInspector countOI) throws HiveException {
    final WritableDoubleObjectInspector sumElemOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    final WritableLongObjectInspector countElemOI =
            PrimitiveObjectInspectorFactory.writableLongObjectInspector;

    if (o_size != _size) {
        if (_size == -1) {
            init(o_size);
        } else {
            throw new HiveException("Mismatch in the number of elements");
        }
    }
    final double[] sum = _sum;
    final long[] count = _count;
    for (int i = 0, len = _size; i < len; i++) {
        Object sum_e = sumOI.getListElement(o_sum, i);
        sum[i] += sumElemOI.get(sum_e);
        Object count_e = countOI.getListElement(o_count, i);
        count[i] += countElemOI.get(count_e);
    }
}
 
Example #4
Source File: FMPredictGenericUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
void merge(final double o_ret, @Nullable final Object o_sumVjXj,
        @Nullable final Object o_sumV2X2,
        @Nonnull final StandardListObjectInspector sumVjXjOI,
        @Nonnull final StandardListObjectInspector sumV2X2OI) throws HiveException {
    this.ret += o_ret;
    if (o_sumVjXj == null) {
        return;
    }

    if (o_sumV2X2 == null) {//sanity check
        throw new HiveException("o_sumV2X2 should not be null");
    }

    final int factors = sumVjXjOI.getListLength(o_sumVjXj);
    if (sumVjXj == null) {
        this.sumVjXj = new double[factors];
        this.sumV2X2 = new double[factors];
    } else if (sumVjXj.length != factors) {//sanity check
        throw new HiveException("Mismatch in the number of factors");
    }

    final WritableDoubleObjectInspector doubleOI =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    for (int f = 0; f < factors; f++) {
        Object o1 = sumVjXjOI.getListElement(o_sumVjXj, f);
        Object o2 = sumV2X2OI.getListElement(o_sumV2X2, f);
        double d1 = doubleOI.get(o1);
        double d2 = doubleOI.get(o2);
        sumVjXj[f] += d1;
        sumV2X2[f] += d2;
    }
}
 
Example #5
Source File: ArrayNullsRemoverGenericUDF.java    From occurrence with Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 1) {
    throw new UDFArgumentException("removeNulls takes an array as argument");
  }
  if (arguments[0].getCategory() != Category.LIST) {
    throw new UDFArgumentException("removeNulls takes an array as argument");
  }
  retValInspector = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(arguments[0]);
  if (retValInspector.getListElementObjectInspector().getCategory() != Category.PRIMITIVE) {
    primitiveObjectInspector = (PrimitiveObjectInspector) retValInspector.getListElementObjectInspector();
  }
  return retValInspector;
}
 
Example #6
Source File: BuildBinsUDAF.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] OIs) throws HiveException {
    super.init(mode, OIs);

    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
        weightOI = HiveUtils.asDoubleCompatibleOI(OIs[0]);

        // set const values
        nBins = HiveUtils.getConstInt(OIs[1]);
        if (OIs.length == 3) {
            autoShrink = HiveUtils.getConstBoolean(OIs[2]);
        }

        // check value of `num_of_bins`
        if (nBins < 2) {
            throw new UDFArgumentException(
                "Only greater than or equal to 2 is accepted but " + nBins
                        + " was passed as `num_of_bins`.");
        }

        quantiles = getQuantiles();
    } else {
        structOI = (StructObjectInspector) OIs[0];
        autoShrinkField = structOI.getStructFieldRef("autoShrink");
        histogramField = structOI.getStructFieldRef("histogram");
        quantilesField = structOI.getStructFieldRef("quantiles");
        autoShrinkOI =
                (WritableBooleanObjectInspector) autoShrinkField.getFieldObjectInspector();
        histogramOI =
                (StandardListObjectInspector) histogramField.getFieldObjectInspector();
        quantilesOI =
                (StandardListObjectInspector) quantilesField.getFieldObjectInspector();
        histogramElOI =
                (WritableDoubleObjectInspector) histogramOI.getListElementObjectInspector();
        quantileOI =
                (WritableDoubleObjectInspector) quantilesOI.getListElementObjectInspector();
    }

    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
        final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
        fieldOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));

        return ObjectInspectorFactory.getStandardStructObjectInspector(
            Arrays.asList("autoShrink", "histogram", "quantiles"), fieldOIs);
    } else {
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    }
}
 
Example #7
Source File: UDAFToOrderedList.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] argOIs) throws HiveException {
    super.init(mode, argOIs);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        // this flag will be used in `processOptions` and `iterate` (= when Mode.PARTIAL1 or Mode.COMPLETE)
        this.sortByKey = (argOIs.length == 2 && !HiveUtils.isConstString(argOIs[1]))
                || (argOIs.length == 3 && HiveUtils.isConstString(argOIs[2]));

        if (sortByKey) {
            this.valueOI = argOIs[0];
            this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[1]);
        } else {
            // sort values by value itself
            this.valueOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
            this.keyOI = HiveUtils.asPrimitiveObjectInspector(argOIs[0]);
        }

        processOptions(argOIs);
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) argOIs[0];
        this.internalMergeOI = soi;

        // re-extract input value OI
        this.valueListField = soi.getStructFieldRef("valueList");
        StandardListObjectInspector valueListOI =
                (StandardListObjectInspector) valueListField.getFieldObjectInspector();
        this.valueOI = valueListOI.getListElementObjectInspector();
        this.valueListOI = ObjectInspectorFactory.getStandardListObjectInspector(valueOI);

        // re-extract input key OI
        this.keyListField = soi.getStructFieldRef("keyList");
        StandardListObjectInspector keyListOI =
                (StandardListObjectInspector) keyListField.getFieldObjectInspector();
        this.keyOI = HiveUtils.asPrimitiveObjectInspector(
            keyListOI.getListElementObjectInspector());
        this.keyListOI = ObjectInspectorFactory.getStandardListObjectInspector(keyOI);

        this.sizeField = soi.getStructFieldRef("size");
        this.reverseOrderField = soi.getStructFieldRef("reverseOrder");

        List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();


        this.outKVField = HiveUtils.getStructFieldRef("outKV", fieldRefs);
        if (outKVField != null) {
            this.outKV = true;
        }
        this.outVKField = HiveUtils.getStructFieldRef("outVK", fieldRefs);
        if (outVKField != null) {
            this.outVK = true;
        }
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI(valueOI, keyOI, outKV, outVK);
    } else {// terminate
        if (outKV) {
            outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(keyOI),
                ObjectInspectorUtils.getStandardObjectInspector(valueOI));
        } else if (outVK) {
            outputOI = ObjectInspectorFactory.getStandardMapObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(valueOI),
                ObjectInspectorUtils.getStandardObjectInspector(keyOI));
        } else {
            outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
                ObjectInspectorUtils.getStandardObjectInspector(valueOI));
        }
    }

    return outputOI;
}