org.apache.hadoop.hive.ql.udf.generic.GenericUDF Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDF. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NamecoinUDFTest.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example #2
Source File: HiveExprAndNode.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example #3
Source File: TryCastUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testList() throws IOException, HiveException {
    // try_cast(array(1.0,2.0,3.0), 'array<string>');
    TryCastUDF udf = new TryCastUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
                TypeInfoFactory.stringTypeInfo, new Text("array<string>"))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    Object result = udf.evaluate(args);

    Assert.assertEquals(WritableUtils.val("0.1", "1.1", "2.1"), result);

    udf.close();
}
 
Example #4
Source File: HiveUDFImplementor.java    From marble with Apache License 2.0 6 votes vote down vote up
public static Object callGenericUDF(GenericUDF udfInstance, Object[] args,
    RelDataTypeHolder[] argsType, ObjectInspector outputObjectInspector) {
  try {
    GenericUDF.DeferredJavaObject[] deferredJavaObjectArray =
        new GenericUDF.DeferredJavaObject[args.length];
    for (int i = 0; i < args.length; i++) {
      deferredJavaObjectArray[i] = new GenericUDF.DeferredJavaObject(
          TypeInferenceUtil.convertCalciteObject2HiveWritableObject(
              argsType[i], args[i]));
    }
    Object result = udfInstance.evaluate(
        deferredJavaObjectArray);
    return TypeInferenceUtil.convertHiveObject2CalciteObject(
        outputObjectInspector,
        result);
  } catch (Exception e) {
    throw new RuntimeException("call hive udf error", e);
  }
}
 
Example #5
Source File: VectorAddUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddDouble() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableFloatObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new float[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Double> expected = Arrays.asList(3.d, 5.d, 7.d);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example #6
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateNullList() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
            new GenericUDF.DeferredJavaObject(new Double(3d))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(Arrays.asList(new DoubleWritable(3d)), result);

    udf.close();
}
 
Example #7
Source File: HiveFunctionRegistry.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
private HiveFuncHolder matchAndCreateUDFHolder(String udfName,
                                               Class<? extends UDF> udfClazz,
                                               CompleteType[] argTypes,
                                               ObjectInspector[] argOIs) {
  try {
    GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz.getName());
    ObjectInspector returnOI = udfInstance.initialize(argOIs);

    return new HiveFuncHolder(
      udfName,
      udfClazz,
      argTypes,
      returnOI,
      CompleteType.fromMinorType(ObjectInspectorHelper.getMinorType(returnOI)),
      nonDeterministicUDFs.contains(udfClazz));
  } catch (Exception e) { /*ignore this*/ }

  return null;
}
 
Example #8
Source File: HiveExprOrNode.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example #9
Source File: UDFStringSplitToMultimapTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}
 
Example #10
Source File: UDFRe2JRegexpExtractAllTest.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Test
public void testUDFRe2JRegexpExtractAll() throws HiveException {
    UDFRe2JRegexpExtractAll udf = new UDFRe2JRegexpExtractAll();

    ObjectInspector source = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector pattern = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] arguments = {source, pattern};

    udf.initialize(arguments);

    GenericUDF.DeferredObject sourceObj = new GenericUDF.DeferredJavaObject("1a 2b 3c 6f");
    GenericUDF.DeferredObject patternObj = new GenericUDF.DeferredJavaObject("\\d+");
    GenericUDF.DeferredObject[] args = {sourceObj, patternObj};

    ArrayList<Object> output = (ArrayList<Object>) udf.evaluate(args);
    assertTrue(Iterables.elementsEqual(ImmutableList.of("1", "2", "3", "6"), output));
}
 
Example #11
Source File: VectorAddUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddLong() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableIntObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new long[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new int[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Long> expected = Arrays.asList(3L, 5L, 7L);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example #12
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateAvoidNullAppend() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(null)};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(3, result.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example #13
Source File: ArrayElementAtUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testString() throws IOException, HiveException {
    ArrayElementAtUDF udf = new ArrayElementAtUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.val("s0", "s1", "s2")),
            new GenericUDF.DeferredJavaObject(1)};

    Assert.assertEquals(WritableUtils.val("s1"), udf.evaluate(args));

    udf.close();
}
 
Example #14
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluate() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(new Double(3))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(4, result.size());
    for (int i = 0; i < 4; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example #15
Source File: NamecoinUDFTest.java    From hadoopcryptoledger with Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
	String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
	byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example #16
Source File: HiveGenericUDF.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void openInternal() {

	LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	try {
		returnInspector = function.initializeAndFoldConstants(
			HiveInspectors.toInspectors(constantArguments, argTypes));
	} catch (UDFArgumentException e) {
		throw new FlinkHiveUDFException(e);
	}

	deferredObjects = new GenericUDF.DeferredObject[argTypes.length];

	for (int i = 0; i < deferredObjects.length; i++) {
		deferredObjects[i] = new DeferredObjectAdapter(
			TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
				HiveTypeUtil.toHiveTypeInfo(argTypes[i])),
			argTypes[i].getLogicalType()
		);
	}
}
 
Example #17
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyMapAndAllNullMap() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<Object, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));
    m.put(null, null);
    arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));

    udf.close();
}
 
Example #18
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test(expected = UDFArgumentException.class)
public void testInvalidMode() throws IOException, HiveException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text("unsupported mode"));
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    udf.evaluate(args);

    udf.close();
}
 
Example #19
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testThreeArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #20
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #21
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumentArray() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #22
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumenString() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #23
Source File: MapRouletteUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example #24
Source File: FeatureHashingUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateList() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        Integer.toString(FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES)),
        "0:1");
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}
 
Example #25
Source File: ToStringArrayUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInputWithNullValue() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add(null);
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example #26
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomK1() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-k1 1.5")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(1.00244958206);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example #27
Source File: OkapiBM25UDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomB() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-b 0.8")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.942443797219);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example #28
Source File: ToStringArrayUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInput() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example #29
Source File: HiveGenericUDF.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void openInternal() {

	LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	ObjectInspector[] argInspectors = HiveInspectors.toInspectors(hiveShim, constantArguments, argTypes);

	try {
		returnInspector = function.initializeAndFoldConstants(argInspectors);
	} catch (UDFArgumentException e) {
		throw new FlinkHiveUDFException(e);
	}

	deferredObjects = new GenericUDF.DeferredObject[argTypes.length];

	for (int i = 0; i < deferredObjects.length; i++) {
		deferredObjects[i] = new DeferredObjectAdapter(
			argInspectors[i],
			argTypes[i].getLogicalType(),
			hiveShim
		);
	}
}
 
Example #30
Source File: VectorDotUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testDotpScalar() throws HiveException, IOException {
    VectorDotUDF udf = new VectorDotUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.writableFloatObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(WritableUtils.val(2.f))};

    Object actual = udf.evaluate(args);
    List<Double> expected = Arrays.asList(2.d, 4.d, 6.d);

    Assert.assertEquals(expected, actual);

    udf.close();
}