Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils#getString()

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Predictor>();
    }
    if (args[1] == null) {// features is null
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Predictor model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.loadPredictor(arg3);
        mapToModel.put(modelId, model);
    }

    Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
    FVec features = denseFeatures ? parseDenseFeatures(args[1])
            : parseSparseFeatures(featureListOI.getList(args[1]));

    predictAndForward(model, rowId, features);
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = PrimitiveObjectInspectorUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float prob = PrimitiveObjectInspectorUtils.getFloat(parameters[3], probOI);

    myAggr.iterate(word, value, label, prob);
}
 

@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer agg,
        Object[] parameters) throws HiveException {
    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;

    if (parameters[0] == null || parameters[1] == null || parameters[2] == null
            || parameters[3] == null) {
        return;
    }

    String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);
    float value = HiveUtils.getFloat(parameters[1], valueOI);
    int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);
    float lambda = HiveUtils.getFloat(parameters[3], lambdaOI);

    myAggr.iterate(word, value, label, lambda);
}
 

@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Booster>();
        this.rowBuffer = new HashMap<String, List<LabeledPointWithRowId>>();
    }
    if (args[1] == null) {
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Booster model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.deserializeBooster(arg3);
        mapToModel.put(modelId, model);
    }

    List<LabeledPointWithRowId> rowBatch = rowBuffer.get(modelId);
    if (rowBatch == null) {
        rowBatch = new ArrayList<LabeledPointWithRowId>(_batchSize);
        rowBuffer.put(modelId, rowBatch);
    }
    LabeledPointWithRowId row = parseRow(args);
    rowBatch.add(row);
    if (rowBatch.size() >= _batchSize) {
        predictAndFlush(model, rowBatch);
    }
}
 

@Override
public List<String> evaluate(@Nonnull final DeferredObject[] arguments) throws HiveException {
    _result.clear();

    final int size = _featureNames.length;
    for (int i = 0; i < size; i++) {
        Object argument = arguments[i + 1].get();
        if (argument == null) {
            if (_emitNull) {
                _result.add(null);
            }
            continue;
        }

        PrimitiveObjectInspector oi = _inputOIs[i];
        String s = PrimitiveObjectInspectorUtils.getString(argument, oi);
        if (s.isEmpty()) {
            if (_emitNull) {
                _result.add(null);
            }
            continue;
        }

        // categorical feature representation
        final String f;
        if (_forceValue) {
            f = _featureNames[i] + '#' + s + ":1";
        } else {
            f = _featureNames[i] + '#' + s;
        }
        _result.add(f);

    }
    return _result;
}
 

@Override
public Object convert(Object input) {
  return PrimitiveObjectInspectorUtils.getString(input, inputOI);
}
 

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
  if (arguments[0] == null || arguments[0].get() == null) {
    return null;
  }

  Object input = arguments[0].get();
  switch(inputType) {
    case BOOLEAN:
      return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
    case BYTE:
      return new Byte(((ByteObjectInspector)argumentOI).get(input));
    case SHORT:
      return new Short(((ShortObjectInspector)argumentOI).get(input));
    case INT:
      return new Integer(((IntObjectInspector)argumentOI).get(input));
    case LONG:
      return new Long(((LongObjectInspector)argumentOI).get(input));
    case FLOAT:
      return new Float(((FloatObjectInspector)argumentOI).get(input));
    case DOUBLE:
      return new Double(((DoubleObjectInspector)argumentOI).get(input));
    case STRING:
      return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
    case BINARY:
      return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
    case VARCHAR:
      if (outputType == PrimitiveCategory.CHAR) {
        HiveVarchar hiveVarchar = PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector) argumentOI);
        return new HiveChar(hiveVarchar.getValue(), HiveChar.MAX_CHAR_LENGTH);
      } else {
        return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
      }
    case CHAR:
      return PrimitiveObjectInspectorUtils.getHiveChar(input, (HiveCharObjectInspector) argumentOI);
    case DATE:
      return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
    case TIMESTAMP:
      return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
    case DECIMAL:
      // return type is a HiveVarchar
      HiveDecimal decimalValue =
          PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
      return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
  }

  throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
}
 

@Override
public List<Text> evaluate(@Nonnull final DeferredObject[] arguments) throws HiveException {
    _result.clear();

    final StringBuilder builder = new StringBuilder(128);
    final int size = _featureNames.length;
    for (int i = 0; i < size; i++) {
        Object argument = arguments[i + 1].get();
        if (argument == null) {
            continue;
        }

        PrimitiveObjectInspector oi = _inputOIs[i];
        String s = PrimitiveObjectInspectorUtils.getString(argument, oi);
        if (s.isEmpty()) {
            continue;
        }
        if (s.indexOf(':') != -1) {
            throw new HiveException("feature index SHOULD NOT include colon: " + s);
        }

        final String featureName = _featureNames[i];
        final String feature = featureName + '#' + s;
        // categorical feature representation
        final String fv;
        if (_mhash) {
            int field =
                    _emitIndices ? i : MurmurHash3.murmurhash3(_featureNames[i], _numFields);
            // +NUM_FIELD to avoid conflict to quantitative features
            int index = MurmurHash3.murmurhash3(feature, _numFeatures) + _numFields;
            fv = builder.append(field).append(':').append(index).append(":1").toString();
            StringUtils.clear(builder);
        } else {
            if (_emitIndices) {
                builder.append(i);
            } else {
                builder.append(featureName);
            }
            fv = builder.append(':').append(feature).append(":1").toString();
            StringUtils.clear(builder);
        }

        _result.add(new Text(fv));
    }
    return _result;
}
 

@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    Object wcListObj = internalMergeOI.getStructFieldData(partial, wcListField);

    List<?> wcListRaw = wcListOI.getList(HiveUtils.castLazyBinaryObject(wcListObj));

    // fix list elements to Java String objects
    int wcListSize = wcListRaw.size();
    List<String> wcList = new ArrayList<String>();
    for (int i = 0; i < wcListSize; i++) {
        wcList.add(PrimitiveObjectInspectorUtils.getString(wcListRaw.get(i), wcListElemOI));
    }

    Object probMapObj = internalMergeOI.getStructFieldData(partial, probMapField);
    Map<?, ?> probMapRaw = probMapOI.getMap(HiveUtils.castLazyBinaryObject(probMapObj));

    Map<String, List<Float>> probMap = new HashMap<String, List<Float>>();
    for (Map.Entry<?, ?> e : probMapRaw.entrySet()) {
        // fix map keys to Java String objects
        String word = PrimitiveObjectInspectorUtils.getString(e.getKey(), probMapKeyOI);

        Object probMapValueObj = e.getValue();
        List<?> probMapValueRaw =
                probMapValueOI.getList(HiveUtils.castLazyBinaryObject(probMapValueObj));

        // fix map values to lists of Java Float objects
        int probMapValueSize = probMapValueRaw.size();
        List<Float> prob_word = new ArrayList<Float>();
        for (int i = 0; i < probMapValueSize; i++) {
            prob_word.add(HiveUtils.getFloat(probMapValueRaw.get(i), probMapValueElemOI));
        }

        probMap.put(word, prob_word);
    }

    // restore options from partial result
    Object topicsObj = internalMergeOI.getStructFieldData(partial, topicsOptionField);
    this.topics = PrimitiveObjectInspectorFactory.writableIntObjectInspector.get(topicsObj);

    Object alphaObj = internalMergeOI.getStructFieldData(partial, alphaOptionField);
    this.alpha = PrimitiveObjectInspectorFactory.writableFloatObjectInspector.get(alphaObj);

    Object deltaObj = internalMergeOI.getStructFieldData(partial, deltaOptionField);
    this.delta =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(deltaObj);

    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;
    myAggr.setOptions(topics, alpha, delta);
    myAggr.merge(wcList, probMap);
}
 

@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    Object wcListObj = internalMergeOI.getStructFieldData(partial, wcListField);

    List<?> wcListRaw = wcListOI.getList(HiveUtils.castLazyBinaryObject(wcListObj));

    // fix list elements to Java String objects
    int wcListSize = wcListRaw.size();
    List<String> wcList = new ArrayList<String>();
    for (int i = 0; i < wcListSize; i++) {
        wcList.add(PrimitiveObjectInspectorUtils.getString(wcListRaw.get(i), wcListElemOI));
    }

    Object lambdaMapObj = internalMergeOI.getStructFieldData(partial, lambdaMapField);
    Map<?, ?> lambdaMapRaw =
            lambdaMapOI.getMap(HiveUtils.castLazyBinaryObject(lambdaMapObj));

    Map<String, List<Float>> lambdaMap = new HashMap<String, List<Float>>();
    for (Map.Entry<?, ?> e : lambdaMapRaw.entrySet()) {
        // fix map keys to Java String objects
        String word = PrimitiveObjectInspectorUtils.getString(e.getKey(), lambdaMapKeyOI);

        Object lambdaMapValueObj = e.getValue();
        List<?> lambdaMapValueRaw =
                lambdaMapValueOI.getList(HiveUtils.castLazyBinaryObject(lambdaMapValueObj));

        // fix map values to lists of Java Float objects
        int lambdaMapValueSize = lambdaMapValueRaw.size();
        List<Float> lambda_word = new ArrayList<Float>();
        for (int i = 0; i < lambdaMapValueSize; i++) {
            lambda_word.add(
                HiveUtils.getFloat(lambdaMapValueRaw.get(i), lambdaMapValueElemOI));
        }

        lambdaMap.put(word, lambda_word);
    }

    // restore options from partial result
    Object topicsObj = internalMergeOI.getStructFieldData(partial, topicsOptionField);
    this.topics = PrimitiveObjectInspectorFactory.writableIntObjectInspector.get(topicsObj);

    Object alphaObj = internalMergeOI.getStructFieldData(partial, alphaOptionField);
    this.alpha = PrimitiveObjectInspectorFactory.writableFloatObjectInspector.get(alphaObj);

    Object deltaObj = internalMergeOI.getStructFieldData(partial, deltaOptionField);
    this.delta =
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(deltaObj);

    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;
    myAggr.setOptions(topics, alpha, delta);
    myAggr.merge(wcList, lambdaMap);
}