org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector Java Examples

@VisibleForTesting
public static Block serializeObject(Type type, BlockBuilder builder, Object object, ObjectInspector inspector, boolean filterNullMapKeys)
{
    switch (inspector.getCategory()) {
        case PRIMITIVE:
            serializePrimitive(type, builder, object, (PrimitiveObjectInspector) inspector);
            return null;
        case LIST:
            return serializeList(type, builder, object, (ListObjectInspector) inspector);
        case MAP:
            return serializeMap(type, builder, object, (MapObjectInspector) inspector, filterNullMapKeys);
        case STRUCT:
            return serializeStruct(type, builder, object, (StructObjectInspector) inspector);
        case UNION:
            return serializeUnion(type, builder, object, (UnionObjectInspector) inspector);
    }
    throw new RuntimeException("Unknown object inspector category: " + inspector.getCategory());
}
 

@Test
public void testPA1TrainWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);
}
 

private void writeSingleLevelArray(Object value, ListObjectInspector inspector, GroupType type)
{
    // Get the internal array structure
    Type elementType = type.getType(0);

    recordConsumer.startGroup();

    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(elementType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();

        for (Object element : arrayValues) {
            if (element == null) {
                throw new IllegalArgumentException("Array elements are requires in given schema definition");
            }
            writeValue(element, elementInspector, elementType);
        }

        recordConsumer.endField(elementType.getName(), 0);
    }
    recordConsumer.endGroup();
}
 

private ArrayWritable createArray(final Object obj, final ListObjectInspector inspector)
    throws SerDeException {
  final List<?> sourceArray = inspector.getList(obj);
  final ObjectInspector subInspector = inspector.getListElementObjectInspector();
  final List<Writable> array = new ArrayList<Writable>();
  if (sourceArray != null) {
    for (final Object curObj : sourceArray) {
      final Writable newObj = createObject(curObj, subInspector);
      if (newObj != null) {
        array.add(newObj);
      }
    }
  }
  if (array.size() > 0) {
    final ArrayWritable subArray = new ArrayWritable(array.get(0).getClass(),
        array.toArray(new Writable[array.size()]));
    return new ArrayWritable(Writable.class, new Writable[] {subArray});
  } else {
    return null;
  }
}
 

@Nonnull
private static StructObjectInspector internalMergeOutputOI(
        @CheckForNull PrimitiveObjectInspector[] inputOIs) throws UDFArgumentException {
    Preconditions.checkNotNull(inputOIs);

    final int numOIs = inputOIs.length;
    final List<String> fieldNames = new ArrayList<String>(numOIs);
    final List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(numOIs);
    for (int i = 0; i < numOIs; i++) {
        fieldNames.add("f" + String.valueOf(i));
        ObjectInspector elemOI = ObjectInspectorUtils.getStandardObjectInspector(
            inputOIs[i], ObjectInspectorCopyOption.WRITABLE);
        ListObjectInspector listOI =
                ObjectInspectorFactory.getStandardListObjectInspector(elemOI);
        fieldOIs.add(listOI);
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

/**
 * Deparse a Hive object into a Jackson-serializable object. This uses
 * the ObjectInspector to extract the column data.
 *
 * @param obj - Hive object to deparse
 * @param oi  - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(Object obj, ObjectInspector oi) {
	switch (oi.getCategory()) {
		case LIST:
			return deparseList(obj, (ListObjectInspector) oi);
		case MAP:
			return deparseMap(obj, (MapObjectInspector) oi);
		case PRIMITIVE:
			return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
		case STRUCT:
			return deparseStruct(obj, (StructObjectInspector) oi, false);
		case UNION:
			// Unsupported by JSON
		default:
			return null;
	}
}
 

private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 

private ColumnVector getColumnVector(ObjectInspector oi) {
  Category category = oi.getCategory();
  switch (category) {

    case PRIMITIVE:
      return getPrimitiveColumnVector((PrimitiveObjectInspector)oi);
    case LIST:
      return getListColumnVector((ListObjectInspector)oi);
    case STRUCT:
      return getStructColumnVector((StructObjectInspector)oi);
    case MAP:
      return getMapColumnVector((MapObjectInspector)oi);
    case UNION:
      return getUnionColumnVector((UnionObjectInspector)oi);
    default:
      throw UserException.unsupportedError()
        .message("Vectorized ORC reader is not supported for datatype: %s", category)
        .build(logger);
  }
}
 

/**
 * Deparse a Hive object into a Jackson-serializable object. This uses the
 * ObjectInspector to extract the column data.
 *
 * @param obj
 *            - Hive object to deparse
 * @param oi
 *            - ObjectInspector for the object
 * @return - A deparsed object
 */
private Object deparseObject(final Object obj, final ObjectInspector oi) {
	switch (oi.getCategory()) {
	case PRIMITIVE:
		return deparsePrimitive(obj, (PrimitiveObjectInspector) oi);
	case LIST:
		return deparseList(obj, (ListObjectInspector) oi);
	case MAP:
		return deparseMap(obj, (MapObjectInspector) oi);
	case STRUCT:
		return deparseStruct(obj, (StructObjectInspector) oi, false);
	case UNION:
		// Unsupported by JSON
	default:
		return null;
	}
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 1) {
        throw new UDFArgumentLengthException("The operator 'fallout' accepts 1 argument.");
    }

    // Check that the argument is a list type
    if (arguments[0].getCategory() != ObjectInspector.Category.LIST) {
        throw new UDFArgumentTypeException(1, "Only list type arguments are accepted, but " + arguments[0].getTypeName() + " was passed.");
    }

    // Check that the list is of type long
    // May want to add support for int/double/float later
    switch (((PrimitiveObjectInspector) ((ListObjectInspector) arguments[0]).getListElementObjectInspector()).getPrimitiveCategory()) {
        case LONG:
            break;
        default:
            throw new UDFArgumentTypeException(1, "A long array argument should be passed, but " + arguments[0].getTypeName() + " was passed instead.");
    }

    // Get the list object inspector
    listInputObjectInspector = (ListObjectInspector) arguments[0];

    // This UDF will return a list of doubles
    return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
}
 

/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 

/**
* Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet)
*
* @param loi ObjectInspector for processing the Object containing a list
* @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction
*
* @return a list of BitcoinTransactionOutputs 
*
*/

private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) {
int listLength=loi.getListLength(listOfOutputsObject);
List<BitcoinTransactionOutput> result=new ArrayList<>(listLength);
StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector();
	for (int i=0;i<listLength;i++) {
		Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i);
		StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value");
		StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength");
		StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript");
		if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) {
			LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i);
			return new ArrayList<>();
		}
		HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF));	
		byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF));
		byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF));
		BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript);
		result.add(currentBitcoinTransactionOutput);
	}
return result;
}
 

@Test
public void testPA1TrainWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 0.1");
    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.1000000f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.1000000f, udtf.model.get(3).get(), 1e-5f);
}
 

@Nonnull
private static LabeledPointWithRowId parseDenseFeatures(@Nonnull final Writable rowId,
        @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI,
        @Nonnull final PrimitiveObjectInspector featureElemOI) throws UDFArgumentException {
    final int size = featureListOI.getListLength(argObj);

    final float[] values = new float[size];
    for (int i = 0; i < size; i++) {
        final Object o = featureListOI.getListElement(argObj, i);
        if (o == null) {
            values[i] = Float.NaN;
        } else {
            float v = PrimitiveObjectInspectorUtils.getFloat(o, featureElemOI);
            values[i] = v;
        }
    }

    return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, null, values);

}
 

@Override
public StructObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 4 && argOIs.length != 5) {
        showHelp("Invalid argment length=" + argOIs.length);
    }
    processOptions(argOIs);

    this.rowIdOI = HiveUtils.asPrimitiveObjectInspector(argOIs, 0);
    ListObjectInspector listOI = HiveUtils.asListOI(argOIs, 1);
    this.featureListOI = listOI;
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    if (HiveUtils.isNumberOI(elemOI)) {
        this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
        this.denseFeatures = true;
    } else if (HiveUtils.isStringOI(elemOI)) {
        this.denseFeatures = false;
    } else {
        throw new UDFArgumentException(
            "Expected array<string|double> for the 2nd argment but got an unexpected features type: "
                    + listOI.getTypeName());
    }
    this.modelIdOI = HiveUtils.asStringOI(argOIs, 2);
    this.modelOI = HiveUtils.asStringOI(argOIs, 3);
    return getReturnOI(rowIdOI);
}
 

@Nonnull
private static void serializeList(@Nonnull final StringBuilder sb, @Nullable final Object obj,
        @Nullable final ListObjectInspector loi) throws SerDeException {
    ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
    List<?> olist = loi.getList(obj);

    if (olist == null) {
        sb.append("null");
    } else {
        sb.append(SerDeUtils.LBRACKET);
        for (int i = 0; i < olist.size(); i++) {
            if (i > 0) {
                sb.append(SerDeUtils.COMMA);
            }
            buildJSONString(sb, olist.get(i), listElementObjectInspector);
        }
        sb.append(SerDeUtils.RBRACKET);
    }
}
 

@Test
public void testInspectOptimizerOptions() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        "-opt adam -reg l1 -inspect_opts");

    try {
        udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
        Assert.fail("should not come here");
    } catch (UDFArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("adam"));
    }
}
 

@Nullable
public static ArrayList<Object> copyListObject(@Nonnull final DeferredObject argument,
        @Nonnull final ListObjectInspector loi,
        @Nonnull final ObjectInspectorCopyOption objectInspectorOption) throws HiveException {
    final Object o = argument.get();
    if (o == null) {
        return null;
    }

    final int length = loi.getListLength(o);
    final ArrayList<Object> list = new ArrayList<Object>(length);
    for (int i = 0; i < length; i++) {
        Object e = ObjectInspectorUtils.copyToStandardObject(loi.getListElement(o, i),
            loi.getListElementObjectInspector(), objectInspectorOption);
        list.add(e);
    }
    return list;
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (!HiveUtils.isNumberListOI(argOI0)) {
        throw new UDFArgumentException(
            "Expected array<number> for the first argument: " + argOI0.getTypeName());
    }
    ListObjectInspector xListOI = HiveUtils.asListOI(argOI0);

    ObjectInspector argOI1 = argOIs[1];
    if (HiveUtils.isNumberListOI(argOI1)) {
        this.evaluator = new Dot2DVectors(xListOI, HiveUtils.asListOI(argOI1));
        return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    } else if (HiveUtils.isNumberOI(argOI1)) {
        this.evaluator = new Multiply2D1D(xListOI, argOI1);
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    } else {
        throw new UDFArgumentException(
            "Expected array<number> or number for the send argument: " + argOI1.getTypeName());
    }
}
 

@Nullable
public static double[] asDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull)
        throws UDFArgumentException {
    if (argObj == null) {
        return null;
    }
    final int length = listOI.getListLength(argObj);
    final double[] ary = new double[length];
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return ary;
}
 

@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final boolean avoidNull) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            if (avoidNull) {
                continue;
            }
            throw new UDFArgumentException("Found null at index " + i);
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 

@Nonnull
public static void toDoubleArray(@Nullable final Object argObj,
        @Nonnull final ListObjectInspector listOI,
        @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out,
        final double nullValue) throws UDFArgumentException {
    if (argObj == null) {
        return;
    }
    final int length = listOI.getListLength(argObj);
    if (out.length != length) {
        throw new UDFArgumentException(
            "Dimension mismatched. Expected: " + out.length + ", Actual: " + length);
    }
    for (int i = 0; i < length; i++) {
        Object o = listOI.getListElement(argObj, i);
        if (o == null) {
            out[i] = nullValue;
            continue;
        }
        out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI);
    }
    return;
}
 

ListTreeWriter(int columnId,
               ObjectInspector inspector,
               StreamFactory writer,
               boolean nullable, Configuration conf,
               boolean useVInts, boolean lowMemoryMode,
               MemoryEstimate memoryEstimate) throws IOException {
  super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
  ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
  childrenWriters = new TreeWriter[1];
  childrenWriters[0] =
    createTreeWriter(listObjectInspector.getListElementObjectInspector(),
      writer, true, conf, useVInts, lowMemoryMode, memoryEstimate);
  lengths =
    new RunLengthIntegerWriter(writer.createStream(columnId,
        OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
  recordPosition(rowIndexPosition);
}
 

private static Block serializeList(Type type, BlockBuilder builder, Object object, ListObjectInspector inspector)
{
    List<?> list = inspector.getList(object);
    if (list == null) {
        requireNonNull(builder, "parent builder is null").appendNull();
        return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
        currentBuilder = builder.beginBlockEntry();
    }
    else {
        currentBuilder = elementType.createBlockBuilder(null, list.size());
    }

    for (Object element : list) {
        serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
        builder.closeEntry();
        return null;
    }
    else {
        Block resultBlock = currentBuilder.build();
        return resultBlock;
    }
}
 

public static boolean hasType(ObjectInspector objectInspector, PrimitiveCategory... types)
{
    if (objectInspector instanceof PrimitiveObjectInspector) {
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) objectInspector;
        PrimitiveCategory primitiveCategory = primitiveInspector.getPrimitiveCategory();
        for (PrimitiveCategory type : types) {
            if (primitiveCategory == type) {
                return true;
            }
        }
        return false;
    }
    if (objectInspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) objectInspector;
        return hasType(listInspector.getListElementObjectInspector(), types);
    }
    if (objectInspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) objectInspector;
        return hasType(mapInspector.getMapKeyObjectInspector(), types) ||
                hasType(mapInspector.getMapValueObjectInspector(), types);
    }
    if (objectInspector instanceof StructObjectInspector) {
        for (StructField field : ((StructObjectInspector) objectInspector).getAllStructFieldRefs()) {
            if (hasType(field.getFieldObjectInspector(), types)) {
                return true;
            }
        }
        return false;
    }
    throw new IllegalArgumentException("Unknown object inspector type " + objectInspector);
}
 

@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length < 2) {
        throw new UDFArgumentException(
            "_FUNC_ takes more than 2 arguments: ANY item, Array<Int|BigInt|Text> features [, constant String options]");
    }
    this.itemOI = argOIs[0];

    this.featureListOI = (ListObjectInspector) argOIs[1];
    ObjectInspector featureRawOI = featureListOI.getListElementObjectInspector();
    String keyTypeName = featureRawOI.getTypeName();
    if (!STRING_TYPE_NAME.equals(keyTypeName) && !INT_TYPE_NAME.equals(keyTypeName)
            && !BIGINT_TYPE_NAME.equals(keyTypeName)) {
        throw new UDFArgumentTypeException(0,
            "1st argument must be Map of key type [Int|BitInt|Text]: " + keyTypeName);
    }
    this.parseFeature = STRING_TYPE_NAME.equals(keyTypeName);
    this.forwardObjs = new Object[2];

    processOptions(argOIs);

    ArrayList<String> fieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();

    fieldNames.add("clusterid");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    fieldNames.add("item");
    fieldOIs.add(itemOI);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 

@SuppressWarnings("deprecation")
@Test
public void testInitialize() throws UDFArgumentException {
    AdaGradUDTF udtf = new AdaGradUDTF();
    ObjectInspector labelOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, labelOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, labelOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, labelOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

@Override
public StdArray createArray(StdType stdType, int expectedSize) {
  ListObjectInspector listObjectInspector = (ListObjectInspector) stdType.underlyingType();
  return new HiveArray(
      new ArrayList(expectedSize),
      ObjectInspectorFactory.getStandardListObjectInspector(listObjectInspector.getListElementObjectInspector()),
      this);
}
 

@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    super.init(mode, parameters);

    // Setup the list and element object inspectors.
    listObjectInspector = (ListObjectInspector) parameters[0];
    longObjectInspector = (LongObjectInspector) listObjectInspector.getListElementObjectInspector();

    // Will return a list of longs
    return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
}
 

@Override
void write(Object obj) throws IOException {
    super.write(obj);
    if (obj != null) {
        ListObjectInspector insp = (ListObjectInspector) inspector;
        int len = insp.getListLength(obj);
        lengths.write(len);
        if (createBloomFilter) {
            bloomFilter.addLong(len);
        }
        for (int i = 0; i < len; ++i) {
            childrenWriters[0].write(insp.getListElement(obj, i));
        }
    }
}