Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 2);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.LONG);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 2
Source Project: presto   Source File: GenericHiveRecordCursor.java    License: Apache License 2.0 6 votes vote down vote up
private void parseDecimalColumn(int column)
{
    loaded[column] = true;

    Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);

    if (fieldData == null) {
        nulls[column] = true;
    }
    else {
        Object fieldValue = ((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
        checkState(fieldValue != null, "fieldValue should not be null");

        HiveDecimal decimal = (HiveDecimal) fieldValue;
        DecimalType columnType = (DecimalType) types[column];
        BigInteger unscaledDecimal = rescale(decimal.unscaledValue(), decimal.scale(), columnType.getScale());

        if (columnType.isShort()) {
            longs[column] = unscaledDecimal.longValue();
        }
        else {
            slices[column] = Decimals.encodeUnscaledValue(unscaledDecimal);
        }
        nulls[column] = false;
    }
}
 
Example 3
Source Project: indexr   Source File: IndexRSerde.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Writable serialize(Object obj, ObjectInspector objectInspector) throws SerDeException {
    if (!objectInspector.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        throw new SerDeException("Cannot serialize " + objectInspector.getCategory() + ". Can only serialize a struct");
    }

    StructObjectInspector inspector = (StructObjectInspector) objectInspector;
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    Writable[] arr = new Writable[fields.size()];
    for (int i = 0; i < fields.size(); i++) {
        StructField field = fields.get(i);
        Object subObj = inspector.getStructFieldData(obj, field);
        ObjectInspector subInspector = field.getFieldObjectInspector();
        arr[i] = createPrimitive(subObj, (PrimitiveObjectInspector) subInspector);
    }
    serdeSize = arr.length;
    return new ArrayWritable(Writable.class, arr);
}
 
Example 4
Source Project: incubator-hivemall   Source File: UDAFToOrderedMap.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOI(
        @Nonnull PrimitiveObjectInspector keyOI, @Nonnull ObjectInspector valueOI) {
    List<String> fieldNames = new ArrayList<String>();
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("partialMap");
    fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(
        ObjectInspectorUtils.getStandardObjectInspector(keyOI),
        ObjectInspectorUtils.getStandardObjectInspector(valueOI)));

    fieldNames.add("size");
    fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 5
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  inputObjectInspector = (PrimitiveObjectInspector) parameters[0];

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are partial aggregations.
  if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
    if (parameters.length > 1) {
      kObjectInspector = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 6
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 2) {
        throw new UDFArgumentException(getClass().getSimpleName()
                + " takes 2 arguments: List<Int|BigInt|Text> features, {Int|BitInt|Text} label [, constant text options]");
    }
    PrimitiveObjectInspector featureInputOI = processFeaturesOI(argOIs[0]);
    this.labelInputOI = HiveUtils.asPrimitiveObjectInspector(argOIs[1]);
    String labelTypeName = labelInputOI.getTypeName();
    if (!STRING_TYPE_NAME.equals(labelTypeName) && !INT_TYPE_NAME.equals(labelTypeName)
            && !BIGINT_TYPE_NAME.equals(labelTypeName)) {
        throw new UDFArgumentTypeException(0,
            "label must be a type [Int|BigInt|Text]: " + labelTypeName);
    }

    processOptions(argOIs);

    this.label2model = new HashMap<Object, PredictionModel>(64);
    this.count = 0;

    return getReturnOI(labelInputOI, getFeatureOutputOI(featureInputOI));
}
 
Example 7
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static PrimitiveObjectInspector asFloatingPointOI(@Nonnull final ObjectInspector argOI)
        throws UDFArgumentTypeException {
    if (argOI.getCategory() != Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but "
                + argOI.getTypeName() + " is passed.");
    }
    final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
    switch (oi.getPrimitiveCategory()) {
        case FLOAT:
        case DOUBLE:
        case DECIMAL:
            break;
        default:
            throw new UDFArgumentTypeException(0, "Only floating point number is accepted but "
                    + argOI.getTypeName() + " is passed.");
    }
    return oi;
}
 
Example 8
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static LazySimpleSerDe getLineSerde(@Nonnull final PrimitiveObjectInspector... OIs)
        throws SerDeException {
    if (OIs.length == 0) {
        throw new IllegalArgumentException("OIs must be specified");
    }
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();

    StringBuilder columnNames = new StringBuilder();
    StringBuilder columnTypes = new StringBuilder();
    for (int i = 0; i < OIs.length; i++) {
        columnNames.append('c').append(i + 1).append(',');
        columnTypes.append(OIs[i].getTypeName()).append(',');
    }
    columnNames.deleteCharAt(columnNames.length() - 1);
    columnTypes.deleteCharAt(columnTypes.length() - 1);

    tbl.setProperty("columns", columnNames.toString());
    tbl.setProperty("columns.types", columnTypes.toString());
    serde.initialize(conf, tbl);
    return serde;
}
 
Example 9
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static PrimitiveObjectInspector asIntegerOI(@Nonnull final ObjectInspector argOI)
        throws UDFArgumentTypeException {
    if (argOI.getCategory() != Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but "
                + argOI.getTypeName() + " is passed.");
    }
    final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
    switch (oi.getPrimitiveCategory()) {
        case INT:
        case SHORT:
        case LONG:
        case BYTE:
            break;
        default:
            throw new UDFArgumentTypeException(0,
                "Unexpected type '" + argOI.getTypeName() + "' is passed.");
    }
    return oi;
}
 
Example 10
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static PrimitiveObjectInspector asNumberOI(@Nonnull final ObjectInspector[] argOIs,
        final int argIndex) throws UDFArgumentException {
    final PrimitiveObjectInspector oi = asPrimitiveObjectInspector(argOIs, argIndex);
    switch (oi.getPrimitiveCategory()) {
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case DECIMAL:
            break;
        default:
            throw new UDFArgumentTypeException(argIndex,
                "Only numeric argument is accepted but " + oi.getTypeName() + " is passed.");
    }
    return oi;
}
 
Example 11
@Override
public GenericUDAFEvaluator getEvaluator(final GenericUDAFParameterInfo info)
    throws SemanticException {
  final ObjectInspector[] inspectors = info.getParameterObjectInspectors();
  if (inspectors.length != 2) {
    throw new UDFArgumentException("Two arguments expected");
  }

  if (inspectors[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    throw new UDFArgumentTypeException(0, "Primitive argument expected, but "
        + inspectors[0].getTypeName() + " was recieved");
  }

  if (inspectors[1].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    throw new UDFArgumentTypeException(0, "Primitive argument expected, but "
        + inspectors[1].getTypeName() + " was recieved");
  }
  final PrimitiveObjectInspector inspector2 = (PrimitiveObjectInspector) inspectors[1];
  if (inspector2.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.INT) {
    throw new UDFArgumentTypeException(0, "Integer value expected as the second argument, but "
        + inspector2.getPrimitiveCategory().name() + " was received");
  }

  return createEvaluator();
}
 
Example 12
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters)
    throws HiveException {
  final ObjectInspector result = super.init(mode, parameters);

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are just partial aggregations.
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    if (parameters.length > 1) {
      maxMapSizeObjectInspector = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return result;
}
 
Example 13
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
  final ObjectInspector resultInspector = super.init(mode, inspectors);
  if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
    // input is original data
    if (inspectors.length > 2) {
      summaryModeInspector_ = (PrimitiveObjectInspector) inspectors[2];
    }
  }
  if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
    // intermediate results need to include the nominal number of entries and the summary mode
    return ObjectInspectorFactory.getStandardStructObjectInspector(
      Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SUMMARY_MODE_FIELD, SKETCH_FIELD),
      Arrays.asList(
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.STRING),
        PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY)
      )
    );
  }
  return resultInspector;
}
 
Example 14
static void validateIntegralParameter(final ObjectInspector inspector, final int index)
    throws UDFArgumentTypeException {
  validateCategoryPrimitive(inspector, index);
  final PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) inspector;
  switch (primitiveInspector.getPrimitiveCategory()) {
  case BYTE:
  case SHORT:
  case INT:
  case LONG:
    break;
  // all other types are invalid
  default:
    throw new UDFArgumentTypeException(index, "Only integral type parameters are expected but "
        + primitiveInspector.getPrimitiveCategory().name() + " was passed as parameter " + (index + 1));
  }
}
 
Example 15
Source Project: hiped2   Source File: Geoloc.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 2) {
    throw new UDFArgumentLengthException(
        "The function COUNTRY(ip, geolocfile) takes exactly 2 arguments.");
  }

  converters = new ObjectInspectorConverters.Converter[arguments.length];
  for (int i = 0; i < arguments.length; i++) {
    converters[i] = ObjectInspectorConverters.getConverter(arguments[i],
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
  }

  return PrimitiveObjectInspectorFactory
      .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
}
 
Example 16
Source Project: incubator-hivemall   Source File: HiveJsonStructReader.java    License: Apache License 2.0 6 votes vote down vote up
private Object parsePrimitive(JsonParser parser, PrimitiveObjectInspector oi)
        throws SerDeException, IOException {
    JsonToken currentToken = parser.getCurrentToken();
    if (currentToken == null) {
        return null;
    }
    try {
        switch (parser.getCurrentToken()) {
            case VALUE_FALSE:
            case VALUE_TRUE:
            case VALUE_NUMBER_INT:
            case VALUE_NUMBER_FLOAT:
            case VALUE_STRING:
                return getObjectOfCorrespondingPrimitiveType(parser.getText(), oi);
            case VALUE_NULL:
                return null;
            default:
                throw new SerDeException("unexpected token type: " + currentToken);
        }
    } finally {
        parser.nextToken();
    }
}
 
Example 17
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @return the number of true bits
 */
@Nonnull
public static int setBits(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final BitSet bitset)
        throws UDFArgumentException {
    if (argObj == null) {
        return 0;
    }
    int count = 0;
    final int length = listOI.getListLength(argObj);
    for (int i = 0; i < length; i++) {
        final Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            continue;
        }
        final int index = PrimitiveObjectInspectorUtils.getInt(o, elemOI);
        if (index < 0) {
            throw new UDFArgumentException("Negative index is not allowed: " + index);
        }
        bitset.set(index);
        count++;
    }
    return count;
}
 
Example 18
Source Project: dremio-oss   Source File: HiveORCVectorizedReader.java    License: Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example 19
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 3);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.STRING);

  ObjectInspector inspector3 = ((StructField) fields.get(2)).getFieldObjectInspector();
  Assert.assertEquals(inspector3.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector3 = (PrimitiveObjectInspector) inspector3;
  Assert.assertEquals(primitiveInspector3.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 20
static void checkIntermediateResultInspector(ObjectInspector resultInspector) {
  Assert.assertNotNull(resultInspector);
  Assert.assertEquals(resultInspector.getCategory(), ObjectInspector.Category.STRUCT);
  StructObjectInspector structResultInspector = (StructObjectInspector) resultInspector;
  List<?> fields = structResultInspector.getAllStructFieldRefs();
  Assert.assertEquals(fields.size(), 2);

  ObjectInspector inspector1 = ((StructField) fields.get(0)).getFieldObjectInspector();
  Assert.assertEquals(inspector1.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector1 = (PrimitiveObjectInspector) inspector1;
  Assert.assertEquals(primitiveInspector1.getPrimitiveCategory(), PrimitiveCategory.INT);

  ObjectInspector inspector2 = ((StructField) fields.get(1)).getFieldObjectInspector();
  Assert.assertEquals(inspector2.getCategory(), ObjectInspector.Category.PRIMITIVE);
  PrimitiveObjectInspector primitiveInspector2 = (PrimitiveObjectInspector) inspector2;
  Assert.assertEquals(primitiveInspector2.getPrimitiveCategory(), PrimitiveCategory.BINARY);
}
 
Example 21
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static long[] asLongArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI, @Nonnull PrimitiveObjectInspector elemOI) {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final long[] ary = new long[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            continue;
        }
        ary[i] = PrimitiveObjectInspectorUtils.getLong(o, elemOI);
    }
    return ary;
}
 
Example 22
Source Project: incubator-hivemall   Source File: ArrayAvgGenericUDAF.java    License: Apache License 2.0 6 votes vote down vote up
void doIterate(@Nonnull final Object tuple, @Nonnull ListObjectInspector listOI,
        @Nonnull PrimitiveObjectInspector elemOI) throws HiveException {
    final int size = listOI.getListLength(tuple);
    if (_size == -1) {
        init(size);
    }
    if (size != _size) {// a corner case
        throw new HiveException(
            "Mismatch in the number of elements at tuple: " + tuple.toString());
    }
    final double[] sum = _sum;
    final long[] count = _count;
    for (int i = 0, len = size; i < len; i++) {
        Object o = listOI.getListElement(tuple, i);
        if (o != null) {
            double v = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
            sum[i] += v;
            count[i] += 1L;
        }
    }
}
 
Example 23
Source Project: incubator-hivemall   Source File: GeneralLearnerBaseUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
protected final ObjectInspector getFeatureOutputOI(@Nonnull final FeatureType featureType)
        throws UDFArgumentException {
    final PrimitiveObjectInspector outputOI;
    if (dense_model) {
        // TODO validation
        outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector; // see DenseModel (long/string is also parsed as int)
    } else {
        switch (featureType) {
            case STRING:
                outputOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
                break;
            case INT:
                outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
                break;
            case LONG:
                outputOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
                break;
            default:
                throw new IllegalStateException("Unexpected feature type: " + featureType);
        }
    }
    return outputOI;
}
 
Example 24
Source Project: incubator-hivemall   Source File: SlimUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}
 
Example 25
Source Project: incubator-hivemall   Source File: SlimUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static Int2FloatMap int2floatMap(final int item, @Nonnull final Map<?, ?> map,
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI, @Nullable final FloatMatrix dataMatrix,
        @Nullable Int2FloatMap dst) {
    if (dst == null) {
        dst = new Int2FloatOpenHashMap(map.size());
        dst.defaultReturnValue(0.f);
    } else {
        dst.clear();
    }

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        float rating = PrimitiveObjectInspectorUtils.getFloat(entry.getValue(), valueOI);
        if (rating == 0.f) {
            continue;
        }
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), keyOI);
        dst.put(user, rating);
        if (dataMatrix != null) {
            dataMatrix.set(item, user, rating);
        }
    }

    return dst;
}
 
Example 26
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final boolean avoidNull) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 
Example 27
Source Project: spatial-framework-for-hadoop   Source File: ST_Bin.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] OIs)
		throws UDFArgumentException {
	
	if (OIs.length != 2) {
		throw new UDFArgumentException("Function takes exactly 2 arguments");
	}

	if (OIs[0].getCategory() != Category.PRIMITIVE) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + OIs[0].getCategory());
	}

	oiBinSize = (PrimitiveObjectInspector)OIs[0];
	if (!EnumSet.of(PrimitiveCategory.DECIMAL,PrimitiveCategory.DOUBLE,PrimitiveCategory.INT,PrimitiveCategory.LONG,PrimitiveCategory.SHORT, PrimitiveCategory.FLOAT).contains(oiBinSize.getPrimitiveCategory())) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + oiBinSize.getPrimitiveCategory());
	}

	geomHelper = HiveGeometryOIHelper.create(OIs[1], 1);
	binSizeIsConstant = ObjectInspectorUtils.isConstantObjectInspector(OIs[0]);

	return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
}
 
Example 28
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static boolean isNumberOI(@Nonnull final ObjectInspector argOI) {
    if (argOI.getCategory() != Category.PRIMITIVE) {
        return false;
    }
    final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI;
    switch (oi.getPrimitiveCategory()) {
        case SHORT:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case DECIMAL:
        case BYTE:
            //case TIMESTAMP:
            return true;
        default:
            return false;
    }
}
 
Example 29
Source Project: incubator-hivemall   Source File: LDAPredictUDAFTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testTerminateWithSameTopicProbability() throws Exception {
    udaf = new LDAPredictUDAF();

    inputOIs = new ObjectInspector[] {
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.STRING),
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.INT),
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                PrimitiveObjectInspector.PrimitiveCategory.FLOAT),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-topics 2")};

    evaluator = udaf.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));

    agg = (LDAPredictUDAF.OnlineLDAPredictAggregationBuffer) evaluator.getNewAggregationBuffer();

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    // Assume that all words in a document are NOT in vocabulary that composes a LDA model.
    // Hence, the document should be assigned to topic #1 (#2) with probability 0.5 (0.5).
    for (int i = 0; i < 18; i++) {
        evaluator.iterate(agg, new Object[] {words[i], 0.f, labels[i], lambdas[i]});
    }

    // Probability for each of the two topics should be same.
    List<Object[]> result = (List<Object[]>) evaluator.terminate(agg);
    Assert.assertEquals(result.size(), 2);
    Assert.assertEquals(result.get(0)[1], result.get(1)[1]);
}
 
Example 30
Source Project: elasticsearch-hadoop   Source File: HiveBytesConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void convert(Object from, BytesArray to) {

    Assert.isTrue(from instanceof HiveType,
            String.format("Unexpected object type, expecting [%s], given [%s]", HiveType.class, from.getClass()));

    HiveType ht = (HiveType) from;
    ObjectInspector oi = ht.getObjectInspector();

    Assert.isTrue(Category.STRUCT == oi.getCategory(),
            String.format("Unexpected object category, expecting [%s], given [%s]", Category.STRUCT, oi.getTypeName()));

    StructObjectInspector soi = (StructObjectInspector) oi;
    List<? extends StructField> refs = soi.getAllStructFieldRefs();
    Assert.isTrue(refs.size() == 1, "When using JSON input, only one field is expected");

    StructField structField = refs.get(0);
    ObjectInspector foi = structField.getFieldObjectInspector();

    Assert.isTrue(Category.PRIMITIVE == foi.getCategory(),
            String.format("Unexpected object category, expecting [%s], given [%s]", Category.PRIMITIVE, oi.getTypeName()));

    Object writable = ((PrimitiveObjectInspector) foi).getPrimitiveWritableObject(soi.getStructFieldData(ht.getObject(), structField));

    // HiveVarcharWritable - Hive 0.12+
    if (writable != null && HiveConstants.VARCHAR_WRITABLE.equals(writable.getClass().getName())) {
        // TODO: add dedicated optimization
        to.bytes(writable.toString());
        return;
    }

    super.convert(writable, to);
}