Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDTF

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDTF. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@SuppressWarnings({ "deprecation", "unchecked" })
@Test
public void normalCase() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
  GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
  ObjectInspector resultInspector = func.initialize(inspectors);
  checkResultInspector(resultInspector);
  ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build();
  sketch.update(1, new double[] {1, 2});
  sketch.update(2, new double[] {1, 2});
  MockCollector collector = new MockCollector();
  func.setCollector(collector);
  func.process(new Object[] {new BytesWritable(sketch.toByteArray())});
  Assert.assertEquals(collector.list.size(), 2);
  Assert.assertEquals(((Object[]) collector.list.get(0)).length, 1);
  Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(0))[0]), Arrays.asList(1.0, 2.0));
  Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(1))[0]), Arrays.asList(1.0, 2.0));
}
 
Example 2
Source Project: incubator-hivemall   Source File: TestUtils.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows) throws HiveException {
    final T udtf;
    try {
        udtf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udtf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udtf);
    deserializeObjectByKryo(serialized, clazz);

    udtf.setCollector(new Collector() {
        public void collect(Object input) throws HiveException {
            // noop
        }
    });

    for (Object[] row : rows) {
        udtf.process(row);
    }

    // serialization after processing row
    serialized = serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udtf.close();
}
 
Example 3
Source Project: incubator-hivemall   Source File: TestUtils.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows) throws HiveException {
    final T udtf;
    try {
        udtf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udtf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udtf);
    deserializeObjectByKryo(serialized, clazz);

    udtf.setCollector(new Collector() {
        public void collect(Object input) throws HiveException {
            // noop
        }
    });

    for (Object[] row : rows) {
        udtf.process(row);
    }

    // serialization after processing row
    serialized = serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udtf.close();
}
 
Example 4
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeTooFewInspectors() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 5
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeTooManyInspectors() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, stringInspector, stringInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 6
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongCategoryArg1() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { structInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 7
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongCategoryArg2() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, structInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 8
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongTypeArg1() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, stringInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 9
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
public void initializeWrongTypeArg2() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, binaryInspector };
  GenericUDTF func = new GetFrequentItemsFromStringsSketchUDTF();
  func.initialize(inspectors);
}
 
Example 10
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
 public void initializeNoInspectors() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 
Example 11
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentException.class)
 public void initializeTooManyInspectors() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector, binaryInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 
Example 12
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentTypeException.class)
 public void initializeWrongCategory() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { structInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 
Example 13
@SuppressWarnings("deprecation")
@Test(expectedExceptions = UDFArgumentTypeException.class)
 public void initializeWrongType() throws Exception {
   ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector };
   GenericUDTF func = new ArrayOfDoublesSketchToValuesUDTF();
   func.initialize(inspectors);
 }
 
Example 14
Source Project: flink   Source File: HiveGenericUDTF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
}
 
Example 15
Source Project: flink   Source File: HiveTableFactory.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FunctionDefinition createFunctionDefinition(String name, CatalogFunction catalogFunction) {
	String functionClassName = catalogFunction.getClassName();

	if (Boolean.valueOf(catalogFunction.getProperties().get(CatalogConfig.IS_GENERIC))) {
		throw new TableException(
			String.format("HiveFunctionDefinitionFactory does not support generic functions %s yet", name));
	}

	Class clazz;
	try {
		clazz = Thread.currentThread().getContextClassLoader().loadClass(functionClassName);

		LOG.info("Successfully loaded Hive udf '{}' with class '{}'", name, functionClassName);
	} catch (ClassNotFoundException e) {
		throw new TableException(
			String.format("Failed to initiate an instance of class %s.", functionClassName), e);
	}

	if (UDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveSimpleUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveSimpleUDF(new HiveFunctionWrapper<>(functionClassName))
		);
	} else if (GenericUDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveGenericUDF(new HiveFunctionWrapper<>(functionClassName))
		);
	} else if (GenericUDTF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDTF", name);

		HiveGenericUDTF udtf = new HiveGenericUDTF(new HiveFunctionWrapper<>(functionClassName));

		return new TableFunctionDefinition(
			name,
			udtf,
			GenericTypeInfo.of(Row.class)
		);
	} else if (GenericUDAFResolver2.class.isAssignableFrom(clazz) || UDAF.class.isAssignableFrom(clazz)) {
		HiveGenericUDAF udaf;

		if (GenericUDAFResolver2.class.isAssignableFrom(clazz)) {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with no UDAF bridging and Hive version %s",
				name, hiveVersion);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), false, hiveVersion);
		} else {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with UDAF bridging and Hive version %s",
				name, hiveVersion);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), true, hiveVersion);
		}

		return new AggregateFunctionDefinition(
			name,
			udaf,
			GenericTypeInfo.of(Object.class),
			GenericTypeInfo.of(GenericUDAFEvaluator.AggregationBuffer.class)
		);
	} else {
		throw new IllegalArgumentException(
			String.format("HiveFunctionDefinitionFactory cannot initiate FunctionDefinition for class %s", functionClassName));
	}
}
 
Example 16
Source Project: flink   Source File: HiveGenericUDTFTest.java    License: Apache License 2.0 4 votes vote down vote up
private static HiveGenericUDTF init(Class hiveUdfClass, Object[] constantArgs, DataType[] argTypes) throws Exception {
	HiveFunctionWrapper<GenericUDTF> wrapper = new HiveFunctionWrapper(hiveUdfClass.getName());

	HiveGenericUDTF udf = new HiveGenericUDTF(wrapper);

	udf.setArgumentTypesAndConstants(constantArgs, argTypes);
	udf.getHiveResultType(constantArgs, argTypes);

	ObjectInspector[] argumentInspectors = HiveInspectors.toInspectors(constantArgs, argTypes);
	ObjectInspector returnInspector = wrapper.createFunction().initialize(argumentInspectors);

	udf.open(null);

	collector = new TestCollector(returnInspector);
	udf.setCollector(collector);

	return udf;
}
 
Example 17
Source Project: flink   Source File: HiveFunctionDefinitionFactory.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Create a FunctionDefinition from a Hive function's class name.
 * Called directly by {@link org.apache.flink.table.module.hive.HiveModule}.
 */
public FunctionDefinition createFunctionDefinitionFromHiveFunction(String name, String functionClassName) {
	Class clazz;
	try {
		clazz = Thread.currentThread().getContextClassLoader().loadClass(functionClassName);

		LOG.info("Successfully loaded Hive udf '{}' with class '{}'", name, functionClassName);
	} catch (ClassNotFoundException e) {
		throw new TableException(
			String.format("Failed to initiate an instance of class %s.", functionClassName), e);
	}

	if (UDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveSimpleUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveSimpleUDF(new HiveFunctionWrapper<>(functionClassName), hiveShim)
		);
	} else if (GenericUDF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDF", name);

		return new ScalarFunctionDefinition(
			name,
			new HiveGenericUDF(new HiveFunctionWrapper<>(functionClassName), hiveShim)
		);
	} else if (GenericUDTF.class.isAssignableFrom(clazz)) {
		LOG.info("Transforming Hive function '{}' into a HiveGenericUDTF", name);

		HiveGenericUDTF udtf = new HiveGenericUDTF(new HiveFunctionWrapper<>(functionClassName), hiveShim);

		return new TableFunctionDefinition(
			name,
			udtf,
			GenericTypeInfo.of(Row.class)
		);
	} else if (GenericUDAFResolver2.class.isAssignableFrom(clazz) || UDAF.class.isAssignableFrom(clazz)) {
		HiveGenericUDAF udaf;

		if (GenericUDAFResolver2.class.isAssignableFrom(clazz)) {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF without UDAF bridging", name);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), false, hiveShim);
		} else {
			LOG.info(
				"Transforming Hive function '{}' into a HiveGenericUDAF with UDAF bridging", name);

			udaf = new HiveGenericUDAF(new HiveFunctionWrapper<>(functionClassName), true, hiveShim);
		}

		return new AggregateFunctionDefinition(
			name,
			udaf,
			GenericTypeInfo.of(Object.class),
			GenericTypeInfo.of(GenericUDAFEvaluator.AggregationBuffer.class)
		);
	} else {
		throw new IllegalArgumentException(
			String.format("HiveFunctionDefinitionFactory cannot initiate FunctionDefinition for class %s", functionClassName));
	}
}
 
Example 18
Source Project: flink   Source File: HiveGenericUDTF.java    License: Apache License 2.0 4 votes vote down vote up
public HiveGenericUDTF(HiveFunctionWrapper<GenericUDTF> hiveFunctionWrapper, HiveShim hiveShim) {
	this.hiveFunctionWrapper = hiveFunctionWrapper;
	this.hiveShim = hiveShim;
}
 
Example 19
Source Project: flink   Source File: HiveGenericUDTFTest.java    License: Apache License 2.0 4 votes vote down vote up
private static HiveGenericUDTF init(Class hiveUdfClass, Object[] constantArgs, DataType[] argTypes) throws Exception {
	HiveFunctionWrapper<GenericUDTF> wrapper = new HiveFunctionWrapper(hiveUdfClass.getName());

	HiveGenericUDTF udf = new HiveGenericUDTF(wrapper, hiveShim);

	udf.setArgumentTypesAndConstants(constantArgs, argTypes);
	udf.getHiveResultType(constantArgs, argTypes);

	ObjectInspector[] argumentInspectors = HiveInspectors.toInspectors(hiveShim, constantArgs, argTypes);
	ObjectInspector returnInspector = wrapper.createFunction().initialize(argumentInspectors);

	udf.open(null);

	collector = new TestCollector(returnInspector);
	udf.setCollector(collector);

	return udf;
}