Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredJavaObject

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredJavaObject . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArrayToStrUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleCase() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaIntObjectInspector),
            PrimitiveObjectInspectorFactory.writableStringObjectInspector});

    Text sep = new Text("#");
    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
                    new GenericUDF.DeferredJavaObject(sep)};
    Assert.assertEquals("1#2#3", udf.evaluate(args));

    args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
            new GenericUDF.DeferredJavaObject(null)};
    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}
 
Example 2
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyMapAndAllNullMap() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<Object, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));
    m.put(null, null);
    arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));

    udf.close();
}
 
Example 3
Source File: VectorAddUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddLong() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableIntObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new long[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new int[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Long> expected = Arrays.asList(3L, 5L, 7L);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example 4
Source File: UDFStringSplitToMultimapTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 5
Source File: ToJsonUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleArray() throws Exception {
    ToJsonUDF udf = new ToJsonUDF();

    ObjectInspector[] argOIs =
            new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    udf.initialize(argOIs);
    Text serialized = udf.evaluate(args);

    Assert.assertEquals("[0.1,1.1,2.1]", serialized.toString());

    udf.close();
}
 
Example 6
Source File: NamecoinUDFTest.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
	String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
	byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 7
Source File: NamecoinUDFTest.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 8
Source File: HiveUDFImplementor.java    From marble with Apache License 2.0 6 votes vote down vote up
public static Object callGenericUDF(GenericUDF udfInstance, Object[] args,
    RelDataTypeHolder[] argsType, ObjectInspector outputObjectInspector) {
  try {
    GenericUDF.DeferredJavaObject[] deferredJavaObjectArray =
        new GenericUDF.DeferredJavaObject[args.length];
    for (int i = 0; i < args.length; i++) {
      deferredJavaObjectArray[i] = new GenericUDF.DeferredJavaObject(
          TypeInferenceUtil.convertCalciteObject2HiveWritableObject(
              argsType[i], args[i]));
    }
    Object result = udfInstance.evaluate(
        deferredJavaObjectArray);
    return TypeInferenceUtil.convertHiveObject2CalciteObject(
        outputObjectInspector,
        result);
  } catch (Exception e) {
    throw new RuntimeException("call hive udf error", e);
  }
}
 
Example 9
Source File: ToStringArrayUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInput() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example 10
Source File: FeatureHashingUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateListLibsvm() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            HiveUtils.getConstStringObjectInspector("-libsvm")});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES) + ":1", "0:1");
    Collections.sort(expected);
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}
 
Example 11
Source File: ArrayElementAtUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testString() throws IOException, HiveException {
    ArrayElementAtUDF udf = new ArrayElementAtUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.val("s0", "s1", "s2")),
            new GenericUDF.DeferredJavaObject(1)};

    Assert.assertEquals(WritableUtils.val("s1"), udf.evaluate(args));

    udf.close();
}
 
Example 12
Source File: LastElementUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testNull() throws IOException, HiveException {
    LastElementUDF udf = new LastElementUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))};

    Assert.assertNull(udf.evaluate(args));

    udf.close();
}
 
Example 13
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testNumDocsWithTermIsLessThanOne() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args =
            new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN,
                    VALID_NUM_DOCS, new GenericUDF.DeferredJavaObject(new Integer(0))};

    udf.evaluate(args);
}
 
Example 14
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = HiveException.class)
public void testAvgDocLengthIsZero() throws Exception {
    initializeUDFWithoutOptions();

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, new GenericUDF.DeferredJavaObject(new Double(0.0)), VALID_NUM_DOCS,
            VALID_NUM_DOCS_WITH_TERM};

    udf.evaluate(args);
}
 
Example 15
Source File: MapKeyValuesUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testStringDouble() throws HiveException, IOException {
    MapKeyValuesUDF udf = new MapKeyValuesUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    Map<String, DoubleWritable> input = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        input.put("k" + i, new DoubleWritable(i));
    }

    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};

    List<Object[]> actual = udf.evaluate(arguments);

    Assert.assertEquals(input.size(), actual.size());
    for (Object[] e : actual) {
        Assert.assertEquals(2, e.length);
        Object v = input.get(e[0]);
        Assert.assertEquals(e[1], v);
    }

    udf.close();
}
 
Example 16
Source File: ArraySliceUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testNullReturn() throws IOException, HiveException {
    ArraySliceUDF udf = new ArraySliceUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    IntWritable offset = new IntWritable();
    IntWritable length = new IntWritable();
    DeferredObject arg1 = new GenericUDF.DeferredJavaObject(offset);
    DeferredObject arg2 = new GenericUDF.DeferredJavaObject(length);

    DeferredObject[] args =
            new DeferredObject[] {
                    new GenericUDF.DeferredJavaObject(Arrays.asList("zero", "one", "two",
                        "three", "four", "five", "six", "seven", "eight", "nine", "ten")),
                    arg1, arg2};


    offset.set(-12);
    length.set(0);
    List<Object> actual = udf.evaluate(args);
    Assert.assertNull(actual);

    udf.close();

}
 
Example 17
Source File: FirstElementUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testNull() throws IOException, HiveException {
    FirstElementUDF udf = new FirstElementUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(new double[] {}))};

    Assert.assertNull(udf.evaluate(args));

    udf.close();
}
 
Example 18
Source File: TestUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
        @Nonnull ObjectInspector[] ois, @Nonnull Object[] row)
        throws HiveException, IOException {
    final T udf;
    try {
        udf = clazz.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw new HiveException(e);
    }

    udf.initialize(ois);

    // serialization after initialization
    byte[] serialized = serializeObjectByKryo(udf);
    deserializeObjectByKryo(serialized, clazz);

    int size = row.length;
    GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
    for (int i = 0; i < size; i++) {
        rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
    }

    udf.evaluate(rowDeferred);

    // serialization after evaluating row
    serialized = serializeObjectByKryo(udf);
    TestUtils.deserializeObjectByKryo(serialized, clazz);

    udf.close();
}
 
Example 19
Source File: ArrayToStrUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoSep() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector)});

    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3))};

    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}
 
Example 20
Source File: ArraySliceUDFTest.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Test
public void testNonNullReturn() throws IOException, HiveException {
    ArraySliceUDF udf = new ArraySliceUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            PrimitiveObjectInspectorFactory.writableIntObjectInspector,
            PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    IntWritable offset = new IntWritable();
    IntWritable length = new IntWritable();
    DeferredObject arg1 = new GenericUDF.DeferredJavaObject(offset);
    DeferredObject arg2 = new GenericUDF.DeferredJavaObject(length);
    DeferredObject nullarg = new GenericUDF.DeferredJavaObject(null);

    DeferredObject[] args =
            new DeferredObject[] {
                    new GenericUDF.DeferredJavaObject(Arrays.asList("zero", "one", "two",
                        "three", "four", "five", "six", "seven", "eight", "nine", "ten")),
                    arg1, arg2};

    offset.set(0);
    length.set(3);
    List<Object> actual = udf.evaluate(args);
    Assert.assertEquals(Arrays.asList("zero", "one", "two"), actual);

    offset.set(1);
    length.set(-2);
    actual = udf.evaluate(args);
    Assert.assertEquals(
        Arrays.asList("one", "two", "three", "four", "five", "six", "seven", "eight"), actual);

    offset.set(1);
    length.set(0);
    actual = udf.evaluate(args);
    Assert.assertEquals(Collections.emptyList(), actual);

    offset.set(-1);
    length.set(0);
    actual = udf.evaluate(args);
    Assert.assertEquals(Collections.emptyList(), actual);

    offset.set(6);
    args[2] = nullarg;
    actual = udf.evaluate(args);
    Assert.assertEquals(Arrays.asList("six", "seven", "eight", "nine", "ten"), actual);

    udf.close();
}