Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#getNewAggregationBuffer()

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#getNewAggregationBuffer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransposeAndDotUDAFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void test() throws Exception {
    final TransposeAndDotUDAF tad = new TransposeAndDotUDAF();

    final double[][] matrix0 = new double[][] {{1, -2}, {-1, 3}};
    final double[][] matrix1 = new double[][] {{1, 2}, {3, 4}};

    final ObjectInspector[] OIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
    final GenericUDAFEvaluator evaluator =
            tad.getEvaluator(new SimpleGenericUDAFParameterInfo(OIs, false, false));
    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, OIs);
    TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer agg =
            (TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer) evaluator.getNewAggregationBuffer();
    evaluator.reset(agg);
    for (int i = 0; i < matrix0.length; i++) {
        evaluator.iterate(agg, new Object[] {WritableUtils.toWritableList(matrix0[i]),
                WritableUtils.toWritableList(matrix1[i])});
    }

    final double[][] answer = new double[][] {{-2.0, -2.0}, {7.0, 8.0}};

    for (int i = 0; i < answer.length; i++) {
        Assert.assertArrayEquals(answer[i], agg.aggMatrix[i], 0.d);
    }
}
 
Example 2
Source File: MergeTest.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartial2() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList);

    // Setup the two partial results
    List<Long> partialResults1 = new ArrayList<>();
    partialResults1.add(1L);
    partialResults1.add(1L);
    partialResults1.add(0L);

    List<Long> partialResults2 = new ArrayList<>();
    partialResults2.add(1L);
    partialResults2.add(1L);
    partialResults2.add(1L);

    // Merge the partial results
    MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, partialResults1);
    udafEvaluator.merge(agg, partialResults2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected results
    List<Long> expected = new ArrayList<>();
    expected.add(2L);
    expected.add(2L);
    expected.add(1L);

    Assert.assertEquals(expected, result);
}
 
Example 3
Source File: MergeTest.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Test(expected = UDFArgumentTypeException.class)
public void testCompleteFunnelSizeMismatch() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    // Setup two funnels, different sizes.
    List<Long> funnel1 = new ArrayList<>();
    funnel1.add(1L);
    funnel1.add(1L);
    funnel1.add(0L);

    List<Long> funnel2 = new ArrayList<>();
    funnel2.add(1L);
    funnel2.add(0L);

    Object[] parameters1 = new Object[]{funnel1};
    Object[] parameters2 = new Object[]{funnel2};

    // Should cause an error when merging funnels of different sizes
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
}
 
Example 4
Source File: MergeTest.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Test(expected = UDFArgumentTypeException.class)
public void testPartial2FunnelSizeMismatch() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList);

    // Setup the two partial results, should fail when merging list of different sizes
    List<Long> partialResults1 = new ArrayList<>();
    partialResults1.add(1L);
    partialResults1.add(1L);

    List<Long> partialResults2 = new ArrayList<>();
    partialResults2.add(1L);
    partialResults2.add(0L);
    partialResults2.add(0L);

    // Merge the partial results, should throw error due to list size difference
    MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, partialResults1);
    udafEvaluator.merge(agg, partialResults2);
}
 
Example 5
Source File: FunnelTest.java    From hive-funnel-udf with Apache License 2.0 4 votes vote down vote up
@Test
public void testComplete() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, new ArrayList<Object>(), Arrays.asList("beta", "BAD"), null, "gamma", Arrays.asList("epsilon")}; // Test empty list funnel step, and null in funnel step
    Object[] parameters2 = new Object[]{"alpha", 100L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 200L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // gamma and beta happen at the same time, beta should come first (sorted on action after timestamp)
    Object[] parameters5 = new Object[]{   null, 800L,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null action_column
    Object[] parameters6 = new Object[]{"omega", null,   Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null timestamp

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    udafEvaluator.iterate(agg, parameters5);
    udafEvaluator.iterate(agg, parameters6);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = new ArrayList<>();
    expected.add(1L);
    expected.add(1L);
    expected.add(0L);

    Assert.assertEquals(expected, result);
}
 
Example 6
Source File: FunnelTest.java    From hive-funnel-udf with Apache License 2.0 4 votes vote down vote up
@Test
public void testPartial1() throws HiveException {
    Funnel udaf = new Funnel();

    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL1, inputObjectInspectorList);

    // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column
    // Funnel is "beta" -> "gamma" -> "epsilon"
    // Should return [1, 1, 0] as we don't have an epsilon
    Object[] parameters1 = new Object[]{ "beta", 200L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters2 = new Object[]{"alpha", 100L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters3 = new Object[]{"delta", 400L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};
    Object[] parameters4 = new Object[]{"gamma", 300L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")};

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    udafEvaluator.iterate(agg, parameters4);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected partial output
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma"));
    expected.add(Arrays.asList(200L, 300L));
    expected.add(Arrays.asList("beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 
Example 7
Source File: FunnelTest.java    From hive-funnel-udf with Apache License 2.0 4 votes vote down vote up
@Test
public void testPartial2() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminatePartial(agg);

    // Expected
    List<Object> expected = new ArrayList<>();
    expected.add(Arrays.asList("beta", "gamma", "alpha"));
    expected.add(Arrays.asList(300L, 400L, 200L));
    expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    Assert.assertEquals(expected, result);
}
 
Example 8
Source File: FunnelTest.java    From hive-funnel-udf with Apache License 2.0 4 votes vote down vote up
@Test
public void testFinal() throws HiveException {
    Funnel udaf = new Funnel();

    // Construct the object inspector for udaf evaluator
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column
        PrimitiveObjectInspectorFactory.javaLongObjectInspector,   // timestamp_column
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1
        ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1
    };
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    // Construct the struct object inspector
    List<String> fieldNames = new ArrayList<>();
    fieldNames.add("action");
    fieldNames.add("timestamp");
    fieldNames.add("funnel");
    List<ObjectInspector> fieldInspectors = new ArrayList<>();
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector));
    fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));
    ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors);
    ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector};

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.FINAL, evaluatorInputObjectInspectorList);

    // Create the two structs to merge
    List<Object> parameter1 = new ArrayList<>();
    parameter1.add(Arrays.asList("beta"));
    parameter1.add(Arrays.asList(300L));
    parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    List<Object> parameter2 = new ArrayList<>();
    parameter2.add(Arrays.asList("gamma", "alpha"));
    parameter2.add(Arrays.asList(400L, 200L));
    parameter2.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null));

    // Process the data
    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.merge(agg, parameter1);
    udafEvaluator.merge(agg, parameter2);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = Arrays.asList(1L, 1L, 1L, 0L);

    Assert.assertEquals(expected, result);
}
 
Example 9
Source File: MergeTest.java    From hive-funnel-udf with Apache License 2.0 4 votes vote down vote up
@Test
public void testComplete() throws HiveException {
    Merge udaf = new Merge();
    ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false);
    GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo);

    ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList);

    List<Long> funnel1 = new ArrayList<>();
    funnel1.add(1L);
    funnel1.add(1L);
    funnel1.add(0L);

    List<Long> funnel2 = new ArrayList<>();
    funnel2.add(1L);
    funnel2.add(0L);
    funnel2.add(0L);

    Object[] parameters1 = new Object[]{null};
    Object[] parameters2 = new Object[]{funnel1};
    Object[] parameters3 = new Object[]{funnel2};

    AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer();
    udafEvaluator.reset(agg);
    udafEvaluator.iterate(agg, parameters1);
    udafEvaluator.iterate(agg, parameters2);
    udafEvaluator.iterate(agg, parameters3);
    Object result = udafEvaluator.terminate(agg);

    // Expected
    List<Long> expected = new ArrayList<>();
    expected.add(2L);
    expected.add(1L);
    expected.add(0L);

    Assert.assertEquals(expected, result);
}