Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#javaLongObjectInspector()

@Nonnull
protected final ObjectInspector getFeatureOutputOI(@Nonnull final FeatureType featureType)
        throws UDFArgumentException {
    final PrimitiveObjectInspector outputOI;
    if (dense_model) {
        // TODO validation
        outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector; // see DenseModel (long/string is also parsed as int)
    } else {
        switch (featureType) {
            case STRING:
                outputOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
                break;
            case INT:
                outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
                break;
            case LONG:
                outputOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
                break;
            default:
                throw new IllegalStateException("Unexpected feature type: " + featureType);
        }
    }
    return outputOI;
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] OIs)
		throws UDFArgumentException {
	
	if (OIs.length != 2) {
		throw new UDFArgumentException("Function takes exactly 2 arguments");
	}

	if (OIs[0].getCategory() != Category.PRIMITIVE) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + OIs[0].getCategory());
	}

	oiBinSize = (PrimitiveObjectInspector)OIs[0];
	if (!EnumSet.of(PrimitiveCategory.DECIMAL,PrimitiveCategory.DOUBLE,PrimitiveCategory.INT,PrimitiveCategory.LONG,PrimitiveCategory.SHORT, PrimitiveCategory.FLOAT).contains(oiBinSize.getPrimitiveCategory())) {
		throw new UDFArgumentException("Argument 0 must be a number - got: " + oiBinSize.getPrimitiveCategory());
	}

	geomHelper = HiveGeometryOIHelper.create(OIs[1], 1);
	binSizeIsConstant = ObjectInspectorUtils.isConstantObjectInspector(OIs[0]);

	return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
}
 

@Test(expected = UDFArgumentTypeException.class)
public void testNonmatchingParamPosition4() throws HiveException {
    Funnel udaf = new Funnel();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaLongObjectInspector
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);
}
 

@Test
public void testInitialize() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, intOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, intOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

@Test
public void testInitialize() throws UDFArgumentException {
    PerceptronUDTF udtf = new PerceptronUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, intOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, intOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

public static ObjectInspector getObjectInspector(int sqlType, String hiveType)
        throws SerDeException {
    switch(sqlType) {
        case Types.VARCHAR:
            return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        case Types.FLOAT:
            return PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
        case Types.DOUBLE:
            return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
        case Types.BOOLEAN:
            return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
        case Types.TINYINT:
            return PrimitiveObjectInspectorFactory.javaByteObjectInspector;
        case Types.SMALLINT:
            return PrimitiveObjectInspectorFactory.javaShortObjectInspector;
        case Types.INTEGER:
            return PrimitiveObjectInspectorFactory.javaIntObjectInspector;
        case Types.BIGINT:
            return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
        case Types.TIMESTAMP:
            return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
        case Types.BINARY:
            return PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;
        case Types.ARRAY:
            String hiveElemType = hiveType.substring(hiveType.indexOf('<') + 1, hiveType.indexOf('>')).trim();
            int sqlElemType = hiveTypeToSqlType(hiveElemType);
            ObjectInspector listElementOI = getObjectInspector(sqlElemType, hiveElemType);
            return ObjectInspectorFactory.getStandardListObjectInspector(listElementOI);
        default:
            throw new SerDeException("Cannot find getObjectInspecto for: " + hiveType);
    }
}
 

@Test
public void testLongInt() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1L),
            new DeferredJavaObject(2)});

    Assert.assertEquals("1:2", ret.toString());
}
 

@Test
public void testLongLong() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1L),
            new DeferredJavaObject(2L)});

    Assert.assertEquals("1:2", ret.toString());
}
 

@Test(expected = UDFArgumentTypeException.class)
public void testBadInputType() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaLongObjectInspector
    };

    udf.initialize(inputOiList);
}
 

@Test(expected = UDFArgumentTypeException.class)
public void testBadInputType() throws HiveException {
    Conversion udf = new Conversion();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaLongObjectInspector
    };

    udf.initialize(inputOiList);
}
 

@Test
public void testStringLong() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject("f1"),
            new DeferredJavaObject(2L)});

    Assert.assertEquals("f1:2", ret.toString());
}
 

@SuppressWarnings("deprecation")
@Test
public void testInitialize() throws UDFArgumentException {
    AdaGradUDTF udtf = new AdaGradUDTF();
    ObjectInspector labelOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI =
            udtf.initialize(new ObjectInspector[] {intListOI, labelOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI =
            udtf.initialize(new ObjectInspector[] {stringListOI, labelOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI =
            udtf.initialize(new ObjectInspector[] {longListOI, labelOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

public static ObjectInspector getObjectInspector(Type kuduType,
                                                 String hiveType) throws SerDeException {
    switch (kuduType) {
        case STRING:
            return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        case FLOAT:
            return PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
        case DOUBLE:
            return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
        case BOOL:
            return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
        case INT8:
            return PrimitiveObjectInspectorFactory.javaByteObjectInspector;
        case INT16:
            return PrimitiveObjectInspectorFactory.javaShortObjectInspector;
        case INT32:
            return PrimitiveObjectInspectorFactory.javaIntObjectInspector;
        case INT64:
            return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
        case TIMESTAMP:
            return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
        case BINARY:
            return PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;
        default:
            throw new SerDeException("Cannot find getObjectInspector for: "
                    + hiveType);
    }
}
 

@Test
public void testPartial1() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL1, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters2 = new Object[]{"alpha", 100L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 300L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected partial output
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma"));
    expected.add(Arrays.asList(200L, 300L));
    expected.add(Arrays.asList("beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 

public static ObjectInspector createObjectInspector(TypeInfo info) {
    switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.javaIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
        case STRING:
          return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        case TIMESTAMP:
          return new PigJodaTimeStampObjectInspector();
        case DECIMAL:
          return new PigDecimalObjectInspector();
        case BINARY:
          return new PigDataByteArrayObjectInspector();
        case DATE:
        case VARCHAR:
        case BYTE:
        case SHORT:
            throw new IllegalArgumentException("Should never happen, " + 
                    (((PrimitiveTypeInfo) info).getPrimitiveCategory()) +
                    "is not valid Pig primitive data type");
        default:
            throw new IllegalArgumentException("Unknown primitive type " +
                    ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new PigStructInspector((StructTypeInfo) info);
    case MAP:
      return new PigMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new PigListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 

@Test
public void testPartial2() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma", "alpha"));
    expected.add(Arrays.asList(300L, 400L, 200L));
    expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 

@Override
protected ObjectInspector createLongType() {
  return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
}
 

@Test
public void testComplete() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, new ArrayList<Object>(), Arrays.asList("beta", "BAD"), null, "gamma", Arrays.asList("epsilon")}; // Test empty list funnel step, and null in funnel step
    Object[] parameters2 = new Object[]{"alpha", 100L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 200L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // gamma and beta happen at the same time, beta should come first (sorted on action after timestamp)
    Object[] parameters5 = new Object[]{   null, 800L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null action_column
    Object[] parameters6 = new Object[]{"omega", null,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null timestamp

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    udafEvaluator.iterate(agg, parameters5);
    udafEvaluator.iterate(agg, parameters6);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = new ArrayList<>();
    expected.add(1L);
    expected.add(1L);
    expected.add(0L);

    Assert.assertEquals(expected, result);
}
 

@Test
public void testLongFeature() throws Exception {
    List<Long> x = Arrays.asList(111L, 222L);
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    testFeature(x, featureOI, Long.class, Long.class);
}
 

@Override
public StdLong createLong(long value) {
  return new HiveLong(value, PrimitiveObjectInspectorFactory.javaLongObjectInspector, this);
}