Java Code Examples for org.apache.hadoop.hive.ql.udf.generic.GenericUDF

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDF. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: HiveGenericUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void openInternal() {

	LOG.info("Open HiveGenericUDF as {}", hiveFunctionWrapper.getClassName());

	function = hiveFunctionWrapper.createFunction();

	try {
		returnInspector = function.initializeAndFoldConstants(
			HiveInspectors.toInspectors(constantArguments, argTypes));
	} catch (UDFArgumentException e) {
		throw new FlinkHiveUDFException(e);
	}

	deferredObjects = new GenericUDF.DeferredObject[argTypes.length];

	for (int i = 0; i < deferredObjects.length; i++) {
		deferredObjects[i] = new DeferredObjectAdapter(
			TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
				HiveTypeUtil.toHiveTypeInfo(argTypes[i])),
			argTypes[i].getLogicalType()
		);
	}
}
 
Example 2
Source Project: incubator-hivemall   Source File: ArrayAppendUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateNullList() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
            new GenericUDF.DeferredJavaObject(new Double(3d))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(Arrays.asList(new DoubleWritable(3d)), result);

    udf.close();
}
 
Example 3
Source Project: marble   Source File: HiveUDFImplementor.java    License: Apache License 2.0 6 votes vote down vote up
public static Object callGenericUDF(GenericUDF udfInstance, Object[] args,
    RelDataTypeHolder[] argsType, ObjectInspector outputObjectInspector) {
  try {
    GenericUDF.DeferredJavaObject[] deferredJavaObjectArray =
        new GenericUDF.DeferredJavaObject[args.length];
    for (int i = 0; i < args.length; i++) {
      deferredJavaObjectArray[i] = new GenericUDF.DeferredJavaObject(
          TypeInferenceUtil.convertCalciteObject2HiveWritableObject(
              argsType[i], args[i]));
    }
    Object result = udfInstance.evaluate(
        deferredJavaObjectArray);
    return TypeInferenceUtil.convertHiveObject2CalciteObject(
        outputObjectInspector,
        result);
  } catch (Exception e) {
    throw new RuntimeException("call hive udf error", e);
  }
}
 
Example 4
Source Project: multiple-dimension-spread   Source File: HiveExprOrNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example 5
Source Project: multiple-dimension-spread   Source File: HiveExprAndNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example 6
Source Project: incubator-hivemall   Source File: VectorDotUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDotp() throws HiveException, IOException {
    VectorDotUDF udf = new VectorDotUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableFloatObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new float[] {2, 3, 4}))};

    Object actual = udf.evaluate(args);
    Double expected = Double.valueOf(1.d * 2.d + 2.d * 3.d + 3.d * 4.d);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example 7
Source Project: dremio-oss   Source File: HiveFunctionRegistry.java    License: Apache License 2.0 6 votes vote down vote up
private HiveFuncHolder matchAndCreateUDFHolder(String udfName,
                                               Class<? extends UDF> udfClazz,
                                               CompleteType[] argTypes,
                                               ObjectInspector[] argOIs) {
  try {
    GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz.getName());
    ObjectInspector returnOI = udfInstance.initialize(argOIs);

    return new HiveFuncHolder(
      udfName,
      udfClazz,
      argTypes,
      returnOI,
      CompleteType.fromMinorType(ObjectInspectorHelper.getMinorType(returnOI)),
      nonDeterministicUDFs.contains(udfClazz));
  } catch (Exception e) { /*ignore this*/ }

  return null;
}
 
Example 8
Source Project: incubator-hivemall   Source File: VectorAddUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddDouble() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableFloatObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new float[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Double> expected = Arrays.asList(3.d, 5.d, 7.d);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example 9
Source Project: incubator-hivemall   Source File: TryCastUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testList() throws IOException, HiveException {
    // try_cast(array(1.0,2.0,3.0), 'array<string>');
    TryCastUDF udf = new TryCastUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
                TypeInfoFactory.stringTypeInfo, new Text("array<string>"))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    Object result = udf.evaluate(args);

    Assert.assertEquals(WritableUtils.val("0.1", "1.1", "2.1"), result);

    udf.close();
}
 
Example 10
@Test
public void testStringSplitToMap() throws Exception {
    UDFStringSplitToMap udf = new UDFStringSplitToMap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, String> output = (HashMap<String, String>) udf.evaluate(args);

    HashMap<String, String> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, String>of("a", "123", "b", "0.4"));

    Assert.assertEquals("split_to_map() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 11
@Test
public void testStringSplitToMultimap() throws Exception {
    UDFStringSplitToMultimap udf = new UDFStringSplitToMultimap();

    GenericUDF.DeferredObject string = new GenericUDF.DeferredJavaObject("a=123,b=0.4,a=124");
    GenericUDF.DeferredObject entryDelimiter = new GenericUDF.DeferredJavaObject(",");
    GenericUDF.DeferredObject keyValueDelimiter = new GenericUDF.DeferredJavaObject("=");
    GenericUDF.DeferredObject[] args = {string, entryDelimiter, keyValueDelimiter};

    HashMap<String, List<String>> output = (HashMap<String, List<String>>) udf.evaluate(args);

    HashMap<String, List<String>> expect = Maps.newHashMap();
    expect.putAll(ImmutableMap.<String, List<String>>of("a", ImmutableList.<String>of("123", "124"), "b", ImmutableList.<String>of("0.4")));

    Assert.assertEquals("split_to_multimap() test", true, MapUtils.mapEquals(output, expect));
}
 
Example 12
@Test
public void testUDFRe2JRegexpExtractAll() throws HiveException {
    UDFRe2JRegexpExtractAll udf = new UDFRe2JRegexpExtractAll();

    ObjectInspector source = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector pattern = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] arguments = {source, pattern};

    udf.initialize(arguments);

    GenericUDF.DeferredObject sourceObj = new GenericUDF.DeferredJavaObject("1a 2b 3c 6f");
    GenericUDF.DeferredObject patternObj = new GenericUDF.DeferredJavaObject("\\d+");
    GenericUDF.DeferredObject[] args = {sourceObj, patternObj};

    ArrayList<Object> output = (ArrayList<Object>) udf.evaluate(args);
    assertTrue(Iterables.elementsEqual(ImmutableList.of("1", "2", "3", "6"), output));
}
 
Example 13
Source Project: incubator-hivemall   Source File: VectorAddUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddLong() throws IOException, HiveException {
    VectorAddUDF udf = new VectorAddUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableIntObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new long[] {1, 2, 3})),
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new int[] {2, 3, 4}))};

    List<?> actual = udf.evaluate(args);
    List<Long> expected = Arrays.asList(3L, 5L, 7L);

    Assert.assertEquals(expected, actual);

    udf.close();
}
 
Example 14
Source Project: incubator-hivemall   Source File: ArrayAppendUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateAvoidNullAppend() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(null)};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(3, result.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example 15
Source Project: incubator-hivemall   Source File: ArrayElementAtUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testString() throws IOException, HiveException {
    ArrayElementAtUDF udf = new ArrayElementAtUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.val("s0", "s1", "s2")),
            new GenericUDF.DeferredJavaObject(1)};

    Assert.assertEquals(WritableUtils.val("s1"), udf.evaluate(args));

    udf.close();
}
 
Example 16
Source Project: incubator-hivemall   Source File: ArrayAppendUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluate() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(new Double(3))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(4, result.size());
    for (int i = 0; i < 4; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example 17
Source Project: hadoopcryptoledger   Source File: NamecoinUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
	String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
	byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 18
Source Project: hadoopcryptoledger   Source File: NamecoinUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldUpdate() throws HiveException {
	String updateScript = "5309642F70616E656C6B612D7B226970223A22382E382E382E38222C226D6170223A7B222A223A7B226970223A22382E382E382E38227D7D7D6D7576A9148D804B079AC79AD0CA108A4E5B679DB591FF069B88AC";
	byte[] updateScriptBytes = BitcoinUtil.convertHexStringToByteArray(updateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(updateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/panelka",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"8.8.8.8\",\"map\":{\"*\":{\"ip\":\"8.8.8.8\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 19
Source Project: incubator-hivemall   Source File: MapRouletteUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyMapAndAllNullMap() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<Object, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)});
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));
    m.put(null, null);
    arguments = new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertNull(udf.evaluate(arguments));

    udf.close();
}
 
Example 20
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = UDFArgumentException.class)
public void testInvalidMode() throws IOException, HiveException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text("unsupported mode"));
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    udf.evaluate(args);

    udf.close();
}
 
Example 21
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testThreeArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 22
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 23
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumentArray() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 24
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumenString() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 25
Source Project: incubator-hivemall   Source File: MapRouletteUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example 26
Source Project: incubator-hivemall   Source File: FeatureHashingUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateList() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        Integer.toString(FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES)),
        "0:1");
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}
 
Example 27
Source Project: incubator-hivemall   Source File: ToStringArrayUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInputWithNullValue() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add(null);
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example 28
Source Project: incubator-hivemall   Source File: OkapiBM25UDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomK1() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-k1 1.5")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(1.00244958206);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example 29
Source Project: incubator-hivemall   Source File: OkapiBM25UDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateWithCustomB() throws Exception {

    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                HiveUtils.getConstStringObjectInspector("-b 0.8")});

    GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] {VALID_TERM_FREQ,
            VALID_DOC_LEN, VALID_AVG_DOC_LEN, VALID_NUM_DOCS, VALID_NUM_DOCS_WITH_TERM};

    DoubleWritable expected = WritableUtils.val(0.942443797219);
    DoubleWritable actual = udf.evaluate(args);
    assertEquals(expected.get(), actual.get(), EPSILON);
}
 
Example 30
Source Project: incubator-hivemall   Source File: ToStringArrayUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInput() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}