Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredObject

@Override
public void openInternal() {

	LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	ObjectInspector[] argInspectors = HiveInspectors.toInspectors(hiveShim, constantArguments, argTypes);

	try {
		returnInspector = function.initializeAndFoldConstants(argInspectors);
	} catch (UDFArgumentException e) {
		throw new FlinkHiveUDFException(e);
	}

	deferredObjects = new GenericUDF.DeferredObject[argTypes.length];

	for (int i = 0; i < deferredObjects.length; i++) {
		deferredObjects[i] = new DeferredObjectAdapter(
			argInspectors[i],
			argTypes[i].getLogicalType(),
			hiveShim
		);
	}
}
 

@Test
public void testStringSplitToMap() throws Exception {
    UDFStringSplitToMap udf = new UDFStringSplitToMap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, String> output = (HashMap<String, String>) udf.evaluate(args);

    HashMap<String, String> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, String>of("a", "123", "b", "0.4"));

    Assert.assertEquals("split_to_map() test", true, MapUtils.mapEquals(output, expect));
}
 

@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}
 

@Test
public void testUDFRe2JRegexpExtractAll() throws HiveException {
    UDFRe2JRegexpExtractAll udf = new UDFRe2JRegexpExtractAll();

    ObjectInspector source = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector pattern = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] arguments = {source, pattern};

    udf.initialize(arguments);

    GenericUDF.DeferredObject sourceObj = new GenericUDF.DeferredJavaObject("1a 2b 3c 6f");
    GenericUDF.DeferredObject patternObj = new GenericUDF.DeferredJavaObject("\\d+");
    GenericUDF.DeferredObject[] args = {sourceObj, patternObj};

    ArrayList<Object> output = (ArrayList<Object>) udf.evaluate(args);
    assertTrue(Iterables.elementsEqual(ImmutableList.of("1", "2", "3", "6"), output));
}
 

@Test
public void testEvaluateWithCustomB() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-b 0.8")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.942443797219);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 

@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 

@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 

@Test
public void testEvaluateWithCustomK1() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-k1 1.5")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(1.00244958206);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 

@Test(expected = HiveException.class)
public void testAvgDocLengthIsZero() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, new GenericUDF.DeferredJavaObject(new Double(0.0)), VALID_NUM_DOCS,
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 

@Test
public void testOnlyOne() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    m.put("One", 324.6);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 

@Test(expected = HiveException.class)
public void testDocLengthIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            new GenericUDF.DeferredJavaObject(new Integer(0)), VALID_AVG_DOC_LEN,
            VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 

@Test(expected = HiveException.class)
public void testTermFrequencyIsNegative() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(new Integer(-1)),
                    VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 

@Test(expected = HiveException.class)
public void testInputArgIsNull() throws Exception {

    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
                    VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 

@Test
public void testEvaluate() throws Exception {

    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.940637195691);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 

public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[] row)
        throws HiveException, IOException {
    final T udf;
    try {
        udf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udf);
    deserializeObjectByKryo(serialized, clazz);

    int size = row.length;
    GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
    for (int i = 0; i < size; i++) {
        rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
    }

    udf.evaluate(rowDeferred);

    // serialization after evaluating row
    serialized = serializeObjectByKryo(udf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udf.close();
}
 

@Test(expected = HiveException.class)
public void testNumDocsIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, new GenericUDF.DeferredJavaObject(new Integer(0)),
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 

public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException {
    UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity();

    ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    ObjectInspector[] arguments = {leftMapOI, rightMapOI};
    udf.initialize(arguments);

    GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap);
    GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap);
    GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj};
    DoubleWritable output = (DoubleWritable) udf.evaluate(args);
    return output == null ? null : output.get();
}
 

@Test
public void testArrayShuffle() throws HiveException {
    UDFArrayShuffle udf = new UDFArrayShuffle();

    ObjectInspector arrayOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    ObjectInspector[] arguments = {arrayOI};

    udf.initialize(arguments);

    List<Integer> array = ImmutableList.of(1,2,5,6);
    GenericUDF.DeferredObject arrayObj = new GenericUDF.DeferredJavaObject(array);
    GenericUDF.DeferredObject[] args = {arrayObj};
    System.out.println(udf.evaluate(args));
}
 

@Override
public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException {
  assert (arguments.length == 2);

  if (arguments[0].get() == null || arguments[1].get() == null) {
    return null;
  }

  String ip = (String) converters[0].convert(arguments[0].get());
  String filename = (String) converters[1].convert(arguments[1].get());

  return lookup(ip, filename);
}
 

@Override
public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException {
    Object array = arguments[0].get();
    int arrayLength = arrayOI.getListLength(array);

    // Check if array is null or empty
    if (array == null || arrayLength <= 0) {
        return null;
    }

    if (arrayLength == 1) {
        return array;
    }

    result.clear();

    if (positions.length < arrayLength) {
        positions = new int[arrayLength];
    }
    for (int i = 0; i < arrayLength; i++) {
        positions[i] = i;
    }

    // Fisher-Yates shuffle
    // Randomly swap a pair of positions
    for (int i = arrayLength - 1; i > 0; i--) {
        Random random = new Random();
        int index = random.nextInt(i + 1);
        int swap = positions[i];
        positions[i] = positions[index];
        positions[index] = swap;
    }

    for (int i = 0; i < arrayLength; i++) {
        Object arrayElement = arrayOI.getListElement(array, positions[i]);
        result.add(arrayElement);
    }
    return result;
}