Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static Block serializeObject(Type type, BlockBuilder builder, Object object, ObjectInspector inspector, boolean filterNullMapKeys)
{
    switch (inspector.getCategory()) {
        case PRIMITIVE:
            serializePrimitive(type, builder, object, (PrimitiveObjectInspector) inspector);
            return null;
        case LIST:
            return serializeList(type, builder, object, (ListObjectInspector) inspector);
        case MAP:
            return serializeMap(type, builder, object, (MapObjectInspector) inspector, filterNullMapKeys);
        case STRUCT:
            return serializeStruct(type, builder, object, (StructObjectInspector) inspector);
        case UNION:
            return serializeUnion(type, builder, object, (UnionObjectInspector) inspector);
    }
    throw new RuntimeException("Unknown object inspector category: " + inspector.getCategory());
}
 
Example 2
@Test
public void testPA1TrainWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);
}
 
Example 3
Source Project: presto   Source File: TestDataWritableWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void writeSingleLevelArray(Object value, ListObjectInspector inspector, GroupType type)
{
    // Get the internal array structure
    Type elementType = type.getType(0);

    recordConsumer.startGroup();

    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(elementType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();

        for (Object element : arrayValues) {
            if (element == null) {
                throw new IllegalArgumentException("Array elements are requires in given schema definition");
            }
            writeValue(element, elementInspector, elementType);
        }

        recordConsumer.endField(elementType.getName(), 0);
    }
    recordConsumer.endGroup();
}
 
Example 4
Source Project: parquet-mr   Source File: ParquetHiveSerDe.java    License: Apache License 2.0 6 votes vote down vote up
private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
    throws SerDeException {
  final List<?> sourceArray = inspector.getList(obj);
  final ObjectInspector subInspector = inspector.getListElementObjectInspector();
  final List<Writable> array = new ArrayList<Writable>();
  if (sourceArray != null) {
    for (final Object curObj : sourceArray) {
      final Writable newObj = createObject(curObj, subInspector);
      if (newObj != null) {
        array.add(newObj);
      }
    }
  }
  if (array.size() > 0) {
    final ArrayWritable subArray = new ArrayWritable(array.get(0).getClass(),
        array.toArray(new Writable[array.size()]));
    return new ArrayWritable(Writable.class, new Writable[] {subArray});
  } else {
    return null;
  }
}
 
Example 5
Source Project: incubator-hivemall   Source File: OnehotEncodingUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 6
Source Project: bigdata-tutorial   Source File: JSONCDHSerDe.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 
Example 7
Source Project: dremio-oss   Source File: HiveORCVectorizedReader.java    License: Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example 8
Source Project: dremio-oss   Source File: HiveORCVectorizedReader.java    License: Apache License 2.0 6 votes vote down vote up
private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 
Example 9
Source Project: searchanalytics-bigdata   Source File: JSONSerDe.java    License: MIT License 6 votes vote down vote up
/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 
Example 10
Source Project: hive-funnel-udf   Source File: Fallout.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 1) {
        throw new UDFArgumentLengthException("The operator 'fallout' accepts 1 argument.");
    }

    // Check that the argument is a list type
    if (arguments[0].getCategory() != ObjectInspector.Category.LIST) {
        throw new UDFArgumentTypeException(1, "Only list type arguments are accepted, but " + arguments[0].getTypeName() + " was passed.");
    }

    // Check that the list is of type long
    // May want to add support for int/double/float later
    switch (((PrimitiveObjectInspector) ((ListObjectInspector) arguments[0]).getListElementObjectInspector()).getPrimitiveCategory()) {
        case LONG:
            break;
        default:
            throw new UDFArgumentTypeException(1, "A long array argument should be passed, but " + arguments[0].getTypeName() + " was passed instead.");
    }

    // Get the list object inspector
    listInputObjectInspector = (ListObjectInspector) arguments[0];

    // This UDF will return a list of doubles
    return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
}
 
Example 11
/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 
Example 12
/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 
Example 13
@Test
public void testPA1TrainWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 0.1");
    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.1000000f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(3).get(), 1e-5f);
}
 
Example 14
@Nonnull
private static LabeledPointWithRowId parseDenseFeatures(@Nonnull final Writable rowId,
        @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI,
        @Nonnull final PrimitiveObjectInspector featureElemOI) throws UDFArgumentException {
    final int size = featureListOI.getListLength(argObj);

    final float[] values = new float[size];
    for (int i = 0; i < size; i++) {
        final Object o = featureListOI.getListElement(argObj, i);
        if (o == null) {
            values[i] = Float.NaN;
        } else {
            float v = PrimitiveObjectInspectorUtils.getFloat(o, featureElemOI);
            values[i] = v;
        }
    }

    return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, null, values);

}
 
Example 15
@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 4 && argOIs.length != 5) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    this.rowIdOI = HiveUtils.asPrimitiveObjectInspector(argOIs, 0);
    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 1);
    this.featureListOI = listOI;
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseFeatures = true;
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.denseFeatures = false;
    } else {
        throw new UDFArgumentException(
            "Expected array<string|double> for the 2nd argment but got an unexpected features type: "
                    + listOI.getTypeName());
    }
    this.modelIdOI = HiveUtils.asStringOI(argOIs, 2);
    this.modelOI = HiveUtils.asStringOI(argOIs, 3);
    return getReturnOI(rowIdOI);
}
 
Example 16
Source Project: incubator-hivemall   Source File: JsonSerdeUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private static void serializeList(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nullable final ListObjectInspector loi) throws SerDeException {
    ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
    List<?> olist = loi.getList(obj);

    if (olist == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACKET);
        for (int i = 0; i < olist.size(); i++) {
            if (i > 0) {
                sb.append(SerDeUtils.COMMA);
            }
            buildJSONString(sb, olist.get(i), listElementObjectInspector);
        }
        sb.append(SerDeUtils.RBRACKET);
    }
}
 
Example 17
@Test
public void testInspectOptimizerOptions() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        "-opt adam -reg l1 -inspect_opts");

    try {
        udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
        Assert.fail("should not come here");
    } catch (UDFArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("adam"));
    }
}
 
Example 18
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
public static ArrayList<Object> copyListObject(@Nonnull final DeferredObject argument,
        @Nonnull final ListObjectInspector loi,
        @Nonnull final ObjectInspectorCopyOption objectInspectorOption) throws HiveException {
    final Object o = argument.get();
    if (o == null) {
        return null;
    }

    final int length = loi.getListLength(o);
    final ArrayList<Object> list = new ArrayList<Object>(length);
    for (int i = 0; i < length; i++) {
        Object e = ObjectInspectorUtils.copyToStandardObject(loi.getListElement(o, i),
            loi.getListElementObjectInspector(), objectInspectorOption);
        list.add(e);
    }
    return list;
}
 
Example 19
Source Project: incubator-hivemall   Source File: VectorDotUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (!HiveUtils.isNumberListOI(argOI0)) {
        throw new UDFArgumentException(
            "Expected array<number> for the first argument: " + argOI0.getTypeName());
    }
    ListObjectInspector xListOI = HiveUtils.asListOI(argOI0);

    ObjectInspector argOI1 = argOIs[1];
    if (HiveUtils.isNumberListOI(argOI1)) {
        this.evaluator = new Dot2DVectors(xListOI, HiveUtils.asListOI(argOI1));
        return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    } else if (HiveUtils.isNumberOI(argOI1)) {
        this.evaluator = new Multiply2D1D(xListOI, argOI1);
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    } else {
        throw new UDFArgumentException(
            "Expected array<number> or number for the send argument: " + argOI1.getTypeName());
    }
}
 
Example 20
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
public static double[] asDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull)
        throws UDFArgumentException {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final double[] ary = new double[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return ary;
}
 
Example 21
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final boolean avoidNull) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 
Example 22
Source Project: incubator-hivemall   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final double nullValue) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            out[i] = nullValue;
            continue;
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 
Example 23
Source Project: hive-dwrf   Source File: WriterImpl.java    License: Apache License 2.0 6 votes vote down vote up
ListTreeWriter(int columnId,
               ObjectInspector inspector,
               StreamFactory writer,
               boolean nullable, Configuration conf,
               boolean useVInts, boolean lowMemoryMode,
               MemoryEstimate memoryEstimate) throws IOException {
  super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
  ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
  childrenWriters = new TreeWriter[1];
  childrenWriters[0] =
    createTreeWriter(listObjectInspector.getListElementObjectInspector(),
      writer, true, conf, useVInts, lowMemoryMode, memoryEstimate);
  lengths =
    new RunLengthIntegerWriter(writer.createStream(columnId,
        OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
  recordPosition(rowIndexPosition);
}
 
Example 24
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static Block serializeList(Type type, BlockBuilder builder, Object object, ListObjectInspector inspector)
{
    List<?> list = inspector.getList(object);
    if (list == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
        currentBuilder = builder.beginBlockEntry();
    }
    else {
        currentBuilder = elementType.createBlockBuilder(null, list.size());
    }

    for (Object element : list) {
        serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
        builder.closeEntry();
        return null;
    }
    else {
        Block resultBlock = currentBuilder.build();
        return resultBlock;
    }
}
 
Example 25
Source Project: presto   Source File: TestHiveFileFormats.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean hasType(ObjectInspector objectInspector, PrimitiveCategory... types)
{
    if (objectInspector instanceof PrimitiveObjectInspector) {
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) objectInspector;
        PrimitiveCategory primitiveCategory = primitiveInspector.getPrimitiveCategory();
        for (PrimitiveCategory type : types) {
            if (primitiveCategory == type) {
                return true;
            }
        }
        return false;
    }
    if (objectInspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) objectInspector;
        return hasType(listInspector.getListElementObjectInspector(), types);
    }
    if (objectInspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) objectInspector;
        return hasType(mapInspector.getMapKeyObjectInspector(), types) ||
                hasType(mapInspector.getMapValueObjectInspector(), types);
    }
    if (objectInspector instanceof StructObjectInspector) {
        for (StructField field : ((StructObjectInspector) objectInspector).getAllStructFieldRefs()) {
            if (hasType(field.getFieldObjectInspector(), types)) {
                return true;
            }
        }
        return false;
    }
    throw new IllegalArgumentException("Unknown object inspector type " + objectInspector);
}
 
Example 26
Source Project: incubator-hivemall   Source File: MinHashUDTF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 2) {
        throw new UDFArgumentException(
            "_FUNC_ takes more than 2 arguments: ANY item, Array<Int|BigInt|Text> features [, constant String options]");
    }
    this.itemOI = argOIs[0];

    this.featureListOI = (ListObjectInspector) argOIs[1];
    ObjectInspector featureRawOI = featureListOI.getListElementObjectInspector();
    String keyTypeName = featureRawOI.getTypeName();
    if (!STRING_TYPE_NAME.equals(keyTypeName) && !INT_TYPE_NAME.equals(keyTypeName)
            && !BIGINT_TYPE_NAME.equals(keyTypeName)) {
        throw new UDFArgumentTypeException(0,
            "1st argument must be Map of key type [Int|BitInt|Text]: " + keyTypeName);
    }
    this.parseFeature = STRING_TYPE_NAME.equals(keyTypeName);
    this.forwardObjs = new Object[2];

    processOptions(argOIs);

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("clusterid");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    fieldNames.add("item");
    fieldOIs.add(itemOI);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 27
Source Project: incubator-hivemall   Source File: AdaGradUDTFTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testInitialize() throws UDFArgumentException {
    AdaGradUDTF udtf = new AdaGradUDTF();
    ObjectInspector labelOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, labelOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, labelOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, labelOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 
Example 28
Source Project: transport   Source File: HiveFactory.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public StdArray createArray(StdType stdType, int expectedSize) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) stdType.underlyingType();
  return new HiveArray(
      new ArrayList(expectedSize),
      ObjectInspectorFactory.getStandardListObjectInspector(listObjectInspector.getListElementObjectInspector()),
      this);
}
 
Example 29
Source Project: hive-funnel-udf   Source File: Merge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    super.init(mode, parameters);

    // Setup the list and element object inspectors.
    listObjectInspector = (ListObjectInspector) parameters[0];
    longObjectInspector = (LongObjectInspector) listObjectInspector.getListElementObjectInspector();

    // Will return a list of longs
    return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
}
 
Example 30
Source Project: localization_nifi   Source File: OrcFlowFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        ListObjectInspector insp = (ListObjectInspector) inspector;
        int len = insp.getListLength(obj);
        lengths.write(len);
        if (createBloomFilter) {
            bloomFilter.addLong(len);
        }
        for (int i = 0; i < len; ++i) {
            childrenWriters[0].write(insp.getListElement(obj, i));
        }
    }
}