Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredObject
The following examples show how to use
org.apache.hadoop.hive.ql.udf.generic.GenericUDF#DeferredObject .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveGenericUDF.java From flink with Apache License 2.0 | 6 votes |
@Override public void openInternal() { LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName()); function = hiveFunctionWrapper.createFunction(); ObjectInspector[] argInspectors = HiveInspectors.toInspectors(hiveShim, constantArguments, argTypes); try { returnInspector = function.initializeAndFoldConstants(argInspectors); } catch (UDFArgumentException e) { throw new FlinkHiveUDFException(e); } deferredObjects = new GenericUDF.DeferredObject[argTypes.length]; for (int i = 0; i < deferredObjects.length; i++) { deferredObjects[i] = new DeferredObjectAdapter( argInspectors[i], argTypes[i].getLogicalType(), hiveShim ); } }
Example 2
Source File: UDFStringSplitToMapTest.java From hive-third-functions with Apache License 2.0 | 6 votes |
@Test public void testStringSplitToMap() throws Exception { UDFStringSplitToMap udf = new UDFStringSplitToMap(); GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4"); GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(","); GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("="); GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter}; HashMap<String, String> output = (HashMap<String, String>) udf.evaluate(args); HashMap<String, String> expect = Maps.newHashMap(); expect.putAll(ImmutableMap.<String, String>of("a", "123", "b", "0.4")); Assert.assertEquals("split_to_map() test", true, MapUtils.mapEquals(output, expect)); }
Example 3
Source File: UDFStringSplitToMultimapTest.java From hive-third-functions with Apache License 2.0 | 6 votes |
@Test public void testStringSplitToMultimap() throws Exception { UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap(); GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124"); GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(","); GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("="); GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter}; HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args); HashMap<String, List<String>> expect = Maps.newHashMap(); expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4"))); Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect)); }
Example 4
Source File: UDFRe2JRegexpExtractAllTest.java From hive-third-functions with Apache License 2.0 | 6 votes |
@Test public void testUDFRe2JRegexpExtractAll() throws HiveException { UDFRe2JRegexpExtractAll udf = new UDFRe2JRegexpExtractAll(); ObjectInspector source = PrimitiveObjectInspectorFactory.javaStringObjectInspector; ObjectInspector pattern = PrimitiveObjectInspectorFactory.javaStringObjectInspector; ObjectInspector[] arguments = {source, pattern}; udf.initialize(arguments); GenericUDF.DeferredObject sourceObj = new GenericUDF.DeferredJavaObject("1a 2b 3c 6f"); GenericUDF.DeferredObject patternObj = new GenericUDF.DeferredJavaObject("\\d+"); GenericUDF.DeferredObject[] args = {sourceObj, patternObj}; ArrayList<Object> output = (ArrayList<Object>) udf.evaluate(args); assertTrue(Iterables.elementsEqual(ImmutableList.of("1", "2", "3", "6"), output)); }
Example 5
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Test public void testEvaluateWithCustomB() throws Exception { udf.initialize( new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, HiveUtils.getConstStringObjectInspector("-b 0.8")}); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; DoubleWritable expected = WritableUtils.val(0.942443797219); DoubleWritable actual = udf.evaluate(args); assertEquals(expected.get(), actual.get(), EPSILON); }
Example 6
Source File: NamecoinUDFTest.java From hadoopcryptoledger with Apache License 2.0 | 6 votes |
@Test public void extractNamecoinFieldUpdate() throws HiveException { String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC"; byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript); NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF(); ObjectInspector[] arguments = new ObjectInspector[1]; arguments[0] = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;; nefu.initialize(arguments); GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1]; doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes)); List<Text> resultList = (List<Text>) nefu.evaluate(doa); Text[] result=resultList.toArray(new Text[resultList.size()]); assertNotNull( result,"Valid result obtained"); // test for domain name assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly"); // test for domain value assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly"); }
Example 7
Source File: MapRouletteUDFTest.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Test public void testSeed() throws HiveException, IOException { MapRouletteUDF udf = new MapRouletteUDF(); Map<String, Double> m = new HashMap<>(); udf.initialize(new ObjectInspector[] { ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector), ObjectInspectorUtils.getConstantObjectInspector( PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)}); m.put("One", 0.7); GenericUDF.DeferredObject[] arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)}; Assert.assertEquals("One", udf.evaluate(arguments)); udf.close(); }
Example 8
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Test public void testEvaluateWithCustomK1() throws Exception { udf.initialize( new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector, HiveUtils.getConstStringObjectInspector("-k1 1.5")}); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; DoubleWritable expected = WritableUtils.val(1.00244958206); DoubleWritable actual = udf.evaluate(args); assertEquals(expected.get(), actual.get(), EPSILON); }
Example 9
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test(expected = HiveException.class) public void testAvgDocLengthIsZero() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, new GenericUDF.DeferredJavaObject(new Double(0.0)), VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; udf.evaluate(args); }
Example 10
Source File: MapRouletteUDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test public void testOnlyOne() throws HiveException, IOException { MapRouletteUDF udf = new MapRouletteUDF(); Map<String, Double> m = new HashMap<>(); udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector( PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)}); m.put("One", 324.6); GenericUDF.DeferredObject[] arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)}; Assert.assertEquals("One", udf.evaluate(arguments)); udf.close(); }
Example 11
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test(expected = HiveException.class) public void testDocLengthIsLessThanOne() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, new GenericUDF.DeferredJavaObject(new Integer(0)), VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; udf.evaluate(args); }
Example 12
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test(expected = HiveException.class) public void testTermFrequencyIsNegative() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(new Integer(-1)), VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; udf.evaluate(args); }
Example 13
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test(expected = HiveException.class) public void testInputArgIsNull() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(null), VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; udf.evaluate(args); }
Example 14
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test public void testEvaluate() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM}; DoubleWritable expected = WritableUtils.val(0.940637195691); DoubleWritable actual = udf.evaluate(args); assertEquals(expected.get(), actual.get(), EPSILON); }
Example 15
Source File: TestUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz, @Nonnull ObjectInspector[] ois, @Nonnull Object[] row) throws HiveException, IOException { final T udf; try { udf = clazz.newInstance(); } catch (InstantiationException | IllegalAccessException e) { throw new HiveException(e); } udf.initialize(ois); // serialization after initialization byte[] serialized = serializeObjectByKryo(udf); deserializeObjectByKryo(serialized, clazz); int size = row.length; GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size]; for (int i = 0; i < size; i++) { rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]); } udf.evaluate(rowDeferred); // serialization after evaluating row serialized = serializeObjectByKryo(udf); TestUtils.deserializeObjectByKryo(serialized, clazz); udf.close(); }
Example 16
Source File: OkapiBM25UDFTest.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Test(expected = HiveException.class) public void testNumDocsIsLessThanOne() throws Exception { initializeUDFWithoutOptions(); GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ, VALID_DOC_LEN, VALID_AVG_DOC_LEN, new GenericUDF.DeferredJavaObject(new Integer(0)), VALID_NUM_DOCS_WITH_TERM}; udf.evaluate(args); }
Example 17
Source File: UDFMathCosineSimilarityTest.java From hive-third-functions with Apache License 2.0 | 5 votes |
public Double getResult(Map<String, Double> leftMap, Map<String, Double> rightMap) throws HiveException { UDFMathCosineSimilarity udf = new UDFMathCosineSimilarity(); ObjectInspector leftMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); ObjectInspector rightMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); ObjectInspector[] arguments = {leftMapOI, rightMapOI}; udf.initialize(arguments); GenericUDF.DeferredObject leftMapObj = new GenericUDF.DeferredJavaObject(leftMap); GenericUDF.DeferredObject rightMapObj = new GenericUDF.DeferredJavaObject(rightMap); GenericUDF.DeferredObject[] args = {leftMapObj, rightMapObj}; DoubleWritable output = (DoubleWritable) udf.evaluate(args); return output == null ? null : output.get(); }
Example 18
Source File: UDFArrayShuffleTest.java From hive-third-functions with Apache License 2.0 | 5 votes |
@Test public void testArrayShuffle() throws HiveException { UDFArrayShuffle udf = new UDFArrayShuffle(); ObjectInspector arrayOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector); ObjectInspector[] arguments = {arrayOI}; udf.initialize(arguments); List<Integer> array = ImmutableList.of(1,2,5,6); GenericUDF.DeferredObject arrayObj = new GenericUDF.DeferredJavaObject(array); GenericUDF.DeferredObject[] args = {arrayObj}; System.out.println(udf.evaluate(args)); }
Example 19
Source File: Geoloc.java From hiped2 with Apache License 2.0 | 5 votes |
@Override public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException { assert (arguments.length == 2); if (arguments[0].get() == null || arguments[1].get() == null) { return null; } String ip = (String) converters[0].convert(arguments[0].get()); String filename = (String) converters[1].convert(arguments[1].get()); return lookup(ip, filename); }
Example 20
Source File: UDFArrayShuffle.java From hive-third-functions with Apache License 2.0 | 5 votes |
@Override public Object evaluate(GenericUDF.DeferredObject[] arguments) throws HiveException { Object array = arguments[0].get(); int arrayLength = arrayOI.getListLength(array); // Check if array is null or empty if (array == null || arrayLength <= 0) { return null; } if (arrayLength == 1) { return array; } result.clear(); if (positions.length < arrayLength) { positions = new int[arrayLength]; } for (int i = 0; i < arrayLength; i++) { positions[i] = i; } // Fisher-Yates shuffle // Randomly swap a pair of positions for (int i = arrayLength - 1; i > 0; i--) { Random random = new Random(); int index = random.nextInt(i + 1); int swap = positions[i]; positions[i] = positions[index]; positions[index] = swap; } for (int i = 0; i < arrayLength; i++) { Object arrayElement = arrayOI.getListElement(array, positions[i]); result.add(arrayElement); } return result; }