Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils#getString()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils#getString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: XGBoostOnlinePredictUDTF.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Predictor>();
    }
    if (args[1] == null) {// features is null
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Predictor model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.loadPredictor(arg3);
        mapToModel.put(modelId, model);
    }

    Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
    FVec features = denseFeatures ? parseDenseFeatures(args[1])
            : parseSparseFeatures(featureListOI.getList(args[1]));

    predictAndForward(model, rowId, features);
}

Example 2

Source File: PLSAPredictUDAF.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = PrimitiveObjectInspectorUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float prob = PrimitiveObjectInspectorUtils.getFloat(parameters[3], probOI);

    myAggr.iterate(word, value, label, prob);
}

Example 3

Source File: LDAPredictUDAF.java From incubator-hivemall with Apache License 2.0

6 votes

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = HiveUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float lambda = HiveUtils.getFloat(parameters[3], lambdaOI);

    myAggr.iterate(word, value, label, lambda);
}

Example 4

Source File: XGBoostBatchPredictUDTF.java From incubator-hivemall with Apache License 2.0

5 votes

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Booster>();
        this.rowBuffer = new HashMap<String, List<LabeledPointWithRowId>>();
    }
    if (args[1] == null) {
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Booster model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.deserializeBooster(arg3);
        mapToModel.put(modelId, model);
    }

    List<LabeledPointWithRowId> rowBatch = rowBuffer.get(modelId);
    if (rowBatch == null) {
        rowBatch = new ArrayList<LabeledPointWithRowId>(_batchSize);
        rowBuffer.put(modelId, rowBatch);
    }
    LabeledPointWithRowId row = parseRow(args);
    rowBatch.add(row);
    if (rowBatch.size() >= _batchSize) {
        predictAndFlush(model, rowBatch);
    }
}

Example 5

Source File: CategoricalFeaturesUDF.java From incubator-hivemall with Apache License 2.0

5 votes

@Override
public List<String> evaluate(@Nonnull final DeferredObject[] arguments) throws HiveException {
    _result.clear();

    final int size = _featureNames.length;
    for (int i = 0; i < size; i++) {
        Object argument = arguments[i + 1].get();
        if (argument == null) {
            if (_emitNull) {
                _result.add(null);
            }
            continue;
        }

        PrimitiveObjectInspector oi = _inputOIs[i];
        String s = PrimitiveObjectInspectorUtils.getString(argument, oi);
        if (s.isEmpty()) {
            if (_emitNull) {
                _result.add(null);
            }
            continue;
        }

        // categorical feature representation
        final String f;
        if (_forceValue) {
            f = _featureNames[i] + '#' + s + ":1";
        } else {
            f = _featureNames[i] + '#' + s;
        }
        _result.add(f);

    }
    return _result;
}

Example 6

Source File: CacheablePrimitiveObjectInspectorConverter.java From transport with BSD 2-Clause "Simplified" License

4 votes

@Override
public Object convert(Object input) {
  return PrimitiveObjectInspectorUtils.getString(input, inputOI);
}

Example 7

Source File: HiveTestUDFImpls.java From dremio-oss with Apache License 2.0

4 votes

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
  if (arguments[0] == null || arguments[0].get() == null) {
    return null;
  }

  Object input = arguments[0].get();
  switch(inputType) {
    case BOOLEAN:
      return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
    case BYTE:
      return new Byte(((ByteObjectInspector)argumentOI).get(input));
    case SHORT:
      return new Short(((ShortObjectInspector)argumentOI).get(input));
    case INT:
      return new Integer(((IntObjectInspector)argumentOI).get(input));
    case LONG:
      return new Long(((LongObjectInspector)argumentOI).get(input));
    case FLOAT:
      return new Float(((FloatObjectInspector)argumentOI).get(input));
    case DOUBLE:
      return new Double(((DoubleObjectInspector)argumentOI).get(input));
    case STRING:
      return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
    case BINARY:
      return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
    case VARCHAR:
      if (outputType == PrimitiveCategory.CHAR) {
        HiveVarchar hiveVarchar = PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector) argumentOI);
        return new HiveChar(hiveVarchar.getValue(), HiveChar.MAX_CHAR_LENGTH);
      } else {
        return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
      }
    case CHAR:
      return PrimitiveObjectInspectorUtils.getHiveChar(input, (HiveCharObjectInspector) argumentOI);
    case DATE:
      return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
    case TIMESTAMP:
      return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
    case DECIMAL:
      // return type is a HiveVarchar
      HiveDecimal decimalValue =
          PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
      return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
  }

  throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
}

Example 8

Source File: FFMFeaturesUDF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public List<Text> evaluate(@Nonnull final DeferredObject[] arguments) throws HiveException {
    _result.clear();

    final StringBuilder builder = new StringBuilder(128);
    final int size = _featureNames.length;
    for (int i = 0; i < size; i++) {
        Object argument = arguments[i + 1].get();
        if (argument == null) {
            continue;
        }

        PrimitiveObjectInspector oi = _inputOIs[i];
        String s = PrimitiveObjectInspectorUtils.getString(argument, oi);
        if (s.isEmpty()) {
            continue;
        }
        if (s.indexOf(':') != -1) {
            throw new HiveException("feature index SHOULD NOT include colon: " + s);
        }

        final String featureName = _featureNames[i];
        final String feature = featureName + '#' + s;
        // categorical feature representation
        final String fv;
        if (_mhash) {
            int field =
                    _emitIndices ? i : MurmurHash3.murmurhash3(_featureNames[i], _numFields);
            // +NUM_FIELD to avoid conflict to quantitative features
            int index = MurmurHash3.murmurhash3(feature, _numFeatures) + _numFields;
            fv = builder.append(field).append(':').append(index).append(":1").toString();
            StringUtils.clear(builder);
        } else {
            if (_emitIndices) {
                builder.append(i);
            } else {
                builder.append(featureName);
            }
            fv = builder.append(':').append(feature).append(":1").toString();
            StringUtils.clear(builder);
        }

        _result.add(new Text(fv));
    }
    return _result;
}

Example 9

Source File: PLSAPredictUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    Object wcListObj = internalMergeOI.getStructFieldData(partial, wcListField);

    List<?> wcListRaw = wcListOI.getList(HiveUtils.castLazyBinaryObject(wcListObj));

    // fix list elements to Java String objects
    int wcListSize = wcListRaw.size();
    List<String> wcList = new ArrayList<String>();
    for (int i = 0; i < wcListSize; i++) {
        wcList.add(PrimitiveObjectInspectorUtils.getString(wcListRaw.get(i), wcListElemOI));
    }

    Object probMapObj = internalMergeOI.getStructFieldData(partial, probMapField);
    Map<?, ?> probMapRaw = probMapOI.getMap(HiveUtils.castLazyBinaryObject(probMapObj));

    Map<String, List<Float>> probMap = new HashMap<String, List<Float>>();
    for (Map.Entry<?, ?> e : probMapRaw.entrySet()) {
        // fix map keys to Java String objects
        String word = PrimitiveObjectInspectorUtils.getString(e.getKey(), probMapKeyOI);

        Object probMapValueObj = e.getValue();
        List<?> probMapValueRaw =
                probMapValueOI.getList(HiveUtils.castLazyBinaryObject(probMapValueObj));

        // fix map values to lists of Java Float objects
        int probMapValueSize = probMapValueRaw.size();
        List<Float> prob_word = new ArrayList<Float>();
        for (int i = 0; i < probMapValueSize; i++) {
            prob_word.add(HiveUtils.getFloat(probMapValueRaw.get(i), probMapValueElemOI));
        }

        probMap.put(word, prob_word);
    }

    // restore options from partial result
    Object topicsObj = internalMergeOI.getStructFieldData(partial, topicsOptionField);
    this.topics = PrimitiveObjectInspectorFactory.writableIntObjectInspector.get(topicsObj);

    Object alphaObj = internalMergeOI.getStructFieldData(partial, alphaOptionField);
    this.alpha = PrimitiveObjectInspectorFactory.writableFloatObjectInspector.get(alphaObj);

    Object deltaObj = internalMergeOI.getStructFieldData(partial, deltaOptionField);
    this.delta =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(deltaObj);

    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;
    myAggr.setOptions(topics, alpha, delta);
    myAggr.merge(wcList, probMap);
}

Example 10

Source File: LDAPredictUDAF.java From incubator-hivemall with Apache License 2.0

4 votes

@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    Object wcListObj = internalMergeOI.getStructFieldData(partial, wcListField);

    List<?> wcListRaw = wcListOI.getList(HiveUtils.castLazyBinaryObject(wcListObj));

    // fix list elements to Java String objects
    int wcListSize = wcListRaw.size();
    List<String> wcList = new ArrayList<String>();
    for (int i = 0; i < wcListSize; i++) {
        wcList.add(PrimitiveObjectInspectorUtils.getString(wcListRaw.get(i), wcListElemOI));
    }

    Object lambdaMapObj = internalMergeOI.getStructFieldData(partial, lambdaMapField);
    Map<?, ?> lambdaMapRaw =
            lambdaMapOI.getMap(HiveUtils.castLazyBinaryObject(lambdaMapObj));

    Map<String, List<Float>> lambdaMap = new HashMap<String, List<Float>>();
    for (Map.Entry<?, ?> e : lambdaMapRaw.entrySet()) {
        // fix map keys to Java String objects
        String word = PrimitiveObjectInspectorUtils.getString(e.getKey(), lambdaMapKeyOI);

        Object lambdaMapValueObj = e.getValue();
        List<?> lambdaMapValueRaw =
                lambdaMapValueOI.getList(HiveUtils.castLazyBinaryObject(lambdaMapValueObj));

        // fix map values to lists of Java Float objects
        int lambdaMapValueSize = lambdaMapValueRaw.size();
        List<Float> lambda_word = new ArrayList<Float>();
        for (int i = 0; i < lambdaMapValueSize; i++) {
            lambda_word.add(
                HiveUtils.getFloat(lambdaMapValueRaw.get(i), lambdaMapValueElemOI));
        }

        lambdaMap.put(word, lambda_word);
    }

    // restore options from partial result
    Object topicsObj = internalMergeOI.getStructFieldData(partial, topicsOptionField);
    this.topics = PrimitiveObjectInspectorFactory.writableIntObjectInspector.get(topicsObj);

    Object alphaObj = internalMergeOI.getStructFieldData(partial, alphaOptionField);
    this.alpha = PrimitiveObjectInspectorFactory.writableFloatObjectInspector.get(alphaObj);

    Object deltaObj = internalMergeOI.getStructFieldData(partial, deltaOptionField);
    this.delta =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(deltaObj);

    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;
    myAggr.setOptions(topics, alpha, delta);
    myAggr.merge(wcList, lambdaMap);
}