Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#getNewAggregationBuffer()
The following examples show how to use
org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator#getNewAggregationBuffer() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransposeAndDotUDAFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test public void test() throws Exception { final TransposeAndDotUDAF tad = new TransposeAndDotUDAF(); final double[][] matrix0 = new double[][] {{1, -2}, {-1, 3}}; final double[][] matrix1 = new double[][] {{1, 2}, {3, 4}}; final ObjectInspector[] OIs = new ObjectInspector[] { ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), ObjectInspectorFactory.getStandardListObjectInspector( PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}; final GenericUDAFEvaluator evaluator = tad.getEvaluator(new SimpleGenericUDAFParameterInfo(OIs, false, false)); evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, OIs); TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer agg = (TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer) evaluator.getNewAggregationBuffer(); evaluator.reset(agg); for (int i = 0; i < matrix0.length; i++) { evaluator.iterate(agg, new Object[] {WritableUtils.toWritableList(matrix0[i]), WritableUtils.toWritableList(matrix1[i])}); } final double[][] answer = new double[][] {{-2.0, -2.0}, {7.0, 8.0}}; for (int i = 0; i < answer.length; i++) { Assert.assertArrayEquals(answer[i], agg.aggMatrix[i], 0.d); } }
Example 2
Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0 | 5 votes |
@Test public void testPartial2() throws HiveException { Merge udaf = new Merge(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector) }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList); // Setup the two partial results List<Long> partialResults1 = new ArrayList<>(); partialResults1.add(1L); partialResults1.add(1L); partialResults1.add(0L); List<Long> partialResults2 = new ArrayList<>(); partialResults2.add(1L); partialResults2.add(1L); partialResults2.add(1L); // Merge the partial results MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.merge(agg, partialResults1); udafEvaluator.merge(agg, partialResults2); Object result = udafEvaluator.terminatePartial(agg); // Expected results List<Long> expected = new ArrayList<>(); expected.add(2L); expected.add(2L); expected.add(1L); Assert.assertEquals(expected, result); }
Example 3
Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0 | 5 votes |
@Test(expected = UDFArgumentTypeException.class) public void testCompleteFunnelSizeMismatch() throws HiveException { Merge udaf = new Merge(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector) }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList); // Setup two funnels, different sizes. List<Long> funnel1 = new ArrayList<>(); funnel1.add(1L); funnel1.add(1L); funnel1.add(0L); List<Long> funnel2 = new ArrayList<>(); funnel2.add(1L); funnel2.add(0L); Object[] parameters1 = new Object[]{funnel1}; Object[] parameters2 = new Object[]{funnel2}; // Should cause an error when merging funnels of different sizes AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.iterate(agg, parameters1); udafEvaluator.iterate(agg, parameters2); }
Example 4
Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0 | 5 votes |
@Test(expected = UDFArgumentTypeException.class) public void testPartial2FunnelSizeMismatch() throws HiveException { Merge udaf = new Merge(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector) }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, inputObjectInspectorList); // Setup the two partial results, should fail when merging list of different sizes List<Long> partialResults1 = new ArrayList<>(); partialResults1.add(1L); partialResults1.add(1L); List<Long> partialResults2 = new ArrayList<>(); partialResults2.add(1L); partialResults2.add(0L); partialResults2.add(0L); // Merge the partial results, should throw error due to list size difference MergeAggregateBuffer agg = (MergeAggregateBuffer) udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.merge(agg, partialResults1); udafEvaluator.merge(agg, partialResults2); }
Example 5
Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testComplete() throws HiveException { Funnel udaf = new Funnel(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column PrimitiveObjectInspectorFactory.javaLongObjectInspector, // timestamp_column ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1 ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1 }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList); // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column // Funnel is "beta" -> "gamma" -> "epsilon" // Should return [1, 1, 0] as we don't have an epsilon Object[] parameters1 = new Object[]{ "beta", 200L, new ArrayList<Object>(), Arrays.asList("beta", "BAD"), null, "gamma", Arrays.asList("epsilon")}; // Test empty list funnel step, and null in funnel step Object[] parameters2 = new Object[]{"alpha", 100L, Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; Object[] parameters3 = new Object[]{"delta", 400L, Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; Object[] parameters4 = new Object[]{"gamma", 200L, Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // gamma and beta happen at the same time, beta should come first (sorted on action after timestamp) Object[] parameters5 = new Object[]{ null, 800L, Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null action_column Object[] parameters6 = new Object[]{"omega", null, Arrays.asList("beta", "BAD"), "gamma", Arrays.asList("epsilon")}; // Check null timestamp // Process the data AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.iterate(agg, parameters1); udafEvaluator.iterate(agg, parameters2); udafEvaluator.iterate(agg, parameters3); udafEvaluator.iterate(agg, parameters4); udafEvaluator.iterate(agg, parameters5); udafEvaluator.iterate(agg, parameters6); Object result = udafEvaluator.terminate(agg); // Expected List<Long> expected = new ArrayList<>(); expected.add(1L); expected.add(1L); expected.add(0L); Assert.assertEquals(expected, result); }
Example 6
Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testPartial1() throws HiveException { Funnel udaf = new Funnel(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column PrimitiveObjectInspectorFactory.javaLongObjectInspector, // timestamp_column ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1 ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1 }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL1, inputObjectInspectorList); // Order will be "alpha, beta, gamma, delta" when ordered on timestamp_column // Funnel is "beta" -> "gamma" -> "epsilon" // Should return [1, 1, 0] as we don't have an epsilon Object[] parameters1 = new Object[]{ "beta", 200L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")}; Object[] parameters2 = new Object[]{"alpha", 100L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")}; Object[] parameters3 = new Object[]{"delta", 400L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")}; Object[] parameters4 = new Object[]{"gamma", 300L, Arrays.asList("beta"), "gamma", Arrays.asList("epsilon")}; // Process the data AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.iterate(agg, parameters1); udafEvaluator.iterate(agg, parameters2); udafEvaluator.iterate(agg, parameters3); udafEvaluator.iterate(agg, parameters4); Object result = udafEvaluator.terminatePartial(agg); // Expected partial output List<Object> expected = new ArrayList<>(); expected.add(Arrays.asList("beta", "gamma")); expected.add(Arrays.asList(200L, 300L)); expected.add(Arrays.asList("beta", null, "gamma", null, "epsilon", null)); Assert.assertEquals(expected, result); }
Example 7
Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testPartial2() throws HiveException { Funnel udaf = new Funnel(); // Construct the object inspector for udaf evaluator ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column PrimitiveObjectInspectorFactory.javaLongObjectInspector, // timestamp_column ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1 ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1 }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); // Construct the struct object inspector List<String> fieldNames = new ArrayList<>(); fieldNames.add("action"); fieldNames.add("timestamp"); fieldNames.add("funnel"); List<ObjectInspector> fieldInspectors = new ArrayList<>(); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors); ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector}; ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.PARTIAL2, evaluatorInputObjectInspectorList); // Create the two structs to merge List<Object> parameter1 = new ArrayList<>(); parameter1.add(Arrays.asList("beta")); parameter1.add(Arrays.asList(300L)); parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); List<Object> parameter2 = new ArrayList<>(); parameter2.add(Arrays.asList("gamma", "alpha")); parameter2.add(Arrays.asList(400L, 200L)); parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); // Process the data AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.merge(agg, parameter1); udafEvaluator.merge(agg, parameter2); Object result = udafEvaluator.terminatePartial(agg); // Expected List<Object> expected = new ArrayList<>(); expected.add(Arrays.asList("beta", "gamma", "alpha")); expected.add(Arrays.asList(300L, 400L, 200L)); expected.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); Assert.assertEquals(expected, result); }
Example 8
Source File: FunnelTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testFinal() throws HiveException { Funnel udaf = new Funnel(); // Construct the object inspector for udaf evaluator ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ PrimitiveObjectInspectorFactory.javaStringObjectInspector, // action_column PrimitiveObjectInspectorFactory.javaLongObjectInspector, // timestamp_column ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), // funnel_step_1 ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector) // funnel_step_1 }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); // Construct the struct object inspector List<String> fieldNames = new ArrayList<>(); fieldNames.add("action"); fieldNames.add("timestamp"); fieldNames.add("funnel"); List<ObjectInspector> fieldInspectors = new ArrayList<>(); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)); fieldInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)); ObjectInspector structObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldInspectors); ObjectInspector[] evaluatorInputObjectInspectorList = new ObjectInspector[]{structObjectInspector}; ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.FINAL, evaluatorInputObjectInspectorList); // Create the two structs to merge List<Object> parameter1 = new ArrayList<>(); parameter1.add(Arrays.asList("beta")); parameter1.add(Arrays.asList(300L)); parameter1.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); List<Object> parameter2 = new ArrayList<>(); parameter2.add(Arrays.asList("gamma", "alpha")); parameter2.add(Arrays.asList(400L, 200L)); parameter2.add(Arrays.asList("alpha", null, "beta", null, "gamma", null, "epsilon", null)); // Process the data AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.merge(agg, parameter1); udafEvaluator.merge(agg, parameter2); Object result = udafEvaluator.terminate(agg); // Expected List<Long> expected = Arrays.asList(1L, 1L, 1L, 0L); Assert.assertEquals(expected, result); }
Example 9
Source File: MergeTest.java From hive-funnel-udf with Apache License 2.0 | 4 votes |
@Test public void testComplete() throws HiveException { Merge udaf = new Merge(); ObjectInspector[] inputObjectInspectorList = new ObjectInspector[]{ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector) }; GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(inputObjectInspectorList, false, false); GenericUDAFEvaluator udafEvaluator = udaf.getEvaluator(paramInfo); ObjectInspector outputObjectInspector = udafEvaluator.init(Mode.COMPLETE, inputObjectInspectorList); List<Long> funnel1 = new ArrayList<>(); funnel1.add(1L); funnel1.add(1L); funnel1.add(0L); List<Long> funnel2 = new ArrayList<>(); funnel2.add(1L); funnel2.add(0L); funnel2.add(0L); Object[] parameters1 = new Object[]{null}; Object[] parameters2 = new Object[]{funnel1}; Object[] parameters3 = new Object[]{funnel2}; AggregationBuffer agg = udafEvaluator.getNewAggregationBuffer(); udafEvaluator.reset(agg); udafEvaluator.iterate(agg, parameters1); udafEvaluator.iterate(agg, parameters2); udafEvaluator.iterate(agg, parameters3); Object result = udafEvaluator.terminate(agg); // Expected List<Long> expected = new ArrayList<>(); expected.add(2L); expected.add(1L); expected.add(0L); Assert.assertEquals(expected, result); }