Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredObject

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredObject . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveGenericUDF.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void openInternal() {

	LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	ObjectInspector[] argInspectors = HiveInspectors.toInspectors(hiveShim, constantArguments, argTypes);

	try {
		returnInspector = function.initializeAndFoldConstants(argInspectors);
	} catch (UDFArgumentException e) {
		throw new FlinkHiveUDFException(e);
	}

	deferredObjects = new GenericUDF.DeferredObject[argTypes.length];

	for (int i = 0; i < deferredObjects.length; i++) {
		deferredObjects[i] = new DeferredObjectAdapter(
			argInspectors[i],
			argTypes[i].getLogicalType(),
			hiveShim
		);
	}
}
 
Example 2
Source File: UDFStringSplitToMapTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringSplitToMap() throws Exception {
    UDFStringSplitToMap udf = new UDFStringSplitToMap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, String> output = (HashMap<String, String>) udf.evaluate(args);

    HashMap<String, String> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, String>of("a", "123", "b", "0.4"));

    Assert.assertEquals("split_to_map() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 3
Source File: UDFStringSplitToMultimapTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 4
Source File: UDFRe2JRegexpExtractAllTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testUDFRe2JRegexpExtractAll() throws HiveException {
    UDFRe2JRegexpExtractAll udf = new UDFRe2JRegexpExtractAll();

    ObjectInspector source = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector pattern = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] arguments = {source, pattern};

    udf.initialize(arguments);

    GenericUDF.DeferredObject sourceObj = new GenericUDF.DeferredJavaObject("1a 2b 3c 6f");
    GenericUDF.DeferredObject patternObj = new GenericUDF.DeferredJavaObject("\\d+");
    GenericUDF.DeferredObject[] args = {sourceObj, patternObj};

    ArrayList<Object> output = (ArrayList<Object>) udf.evaluate(args);
    assertTrue(Iterables.elementsEqual(ImmutableList.of("1", "2", "3", "6"), output));
}
 
Example 5
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomB() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-b 0.8")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.942443797219);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example 6
Source File: NamecoinUDFTest.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 7
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example 8
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomK1() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-k1 1.5")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(1.00244958206);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example 9
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testAvgDocLengthIsZero() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, new GenericUDF.DeferredJavaObject(new Double(0.0)), VALID_NUM_DOCS,
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 10
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testOnlyOne() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    m.put("One", 324.6);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example 11
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testDocLengthIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            new GenericUDF.DeferredJavaObject(new Integer(0)), VALID_AVG_DOC_LEN,
            VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 12
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testTermFrequencyIsNegative() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(new Integer(-1)),
                    VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 13
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testInputArgIsNull() throws Exception {

    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
                    VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 14
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluate() throws Exception {

    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.940637195691);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example 15
Source File: TestUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[] row)
        throws HiveException, IOException {
    final T udf;
    try {
        udf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udf);
    deserializeObjectByKryo(serialized, clazz);

    int size = row.length;
    GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
    for (int i = 0; i < size; i++) {
        rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
    }

    udf.evaluate(rowDeferred);

    // serialization after evaluating row
    serialized = serializeObjectByKryo(udf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udf.close();
}
 
Example 16
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testNumDocsIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, new GenericUDF.DeferredJavaObject(new Integer(0)),
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 17
Source File: UDFMathCosineSimilarityTest.java    From hive-third-functions with Apache License 2.0 5 votes vote down vote up
public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException {
    UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity();

    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap);
    GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap);
    GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj};
    DoubleWritable output = (DoubleWritable) udf.evaluate(args);
    return output == null ? null : output.get();
}
 
Example 18
Source File: UDFArrayShuffleTest.java    From hive-third-functions with Apache License 2.0 5 votes vote down vote up
@Test
public void testArrayShuffle() throws HiveException {
    UDFArrayShuffle udf = new UDFArrayShuffle();

    ObjectInspector arrayOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    ObjectInspector[] arguments = {arrayOI};

    udf.initialize(arguments);

    List<Integer> array = ImmutableList.of(1,2,5,6);
    GenericUDF.DeferredObject arrayObj = new GenericUDF.DeferredJavaObject(array);
    GenericUDF.DeferredObject[] args = {arrayObj};
    System.out.println(udf.evaluate(args));
}
 
Example 19
Source File: Geoloc.java    From hiped2 with Apache License 2.0 5 votes vote down vote up
@Override
public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException {
  assert (arguments.length == 2);

  if (arguments[0].get() == null || arguments[1].get() == null) {
    return null;
  }

  String ip = (String) converters[0].convert(arguments[0].get());
  String filename = (String) converters[1].convert(arguments[1].get());

  return lookup(ip, filename);
}
 
Example 20
Source File: UDFArrayShuffle.java    From hive-third-functions with Apache License 2.0 5 votes vote down vote up
@Override
public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException {
    Object array = arguments[0].get();
    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty
    if (array == null || arrayLength <= 0) {
        return null;
    }

    if (arrayLength == 1) {
        return array;
    }

    result.clear();

    if (positions.length < arrayLength) {
        positions = new int[arrayLength];
    }
    for (int i = 0; i < arrayLength; i++) {
        positions[i] = i;
    }

    // Fisher-Yates shuffle
    // Randomly swap a pair of positions
    for (int i = arrayLength - 1; i > 0; i--) {
        Random random = new Random();
        int index = random.nextInt(i + 1);
        int swap = positions[i];
        positions[i] = positions[index];
        positions[index] = swap;
    }

    for (int i = 0; i < arrayLength; i++) {
        Object arrayElement = arrayOI.getListElement(array, positions[i]);
        result.add(arrayElement);
    }
    return result;
}