Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardConstantListObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory#getStandardConstantListObjectInspector() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testThreeArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 2
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 3
@Test
public void testFiveArgumentArray() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 4
@Test
public void testFiveArgumenString() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 5
@Test
public void testOneArgument() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    List<String> featureNames = Arrays.asList("a");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[2];
    arguments[1] = new DeferredJavaObject(new Double(0.1));

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 6
@Test
public void testTwoArguments() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("1.1");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val("a:0.1", "b:1.1");
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 7
@Test
public void testAvoidZeroWeight() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("0");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 8
@Test
public void testTwoArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new SmartcnUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // stopWords
    argOIs[1] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 9
@Test(expected = UDFArgumentException.class)
public void testEvaluateInvalidUserDictURL() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text("http://google.com/"));
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };

    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);
    Assert.assertNotNull(tokens);

    udf.close();
}
 
Example 10
@Test(expected = UDFArgumentException.class)
public void testMismatch() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b", "c");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    udf.close();
}
 
Example 11
@Test
public void testBooleanWeight() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject(new Boolean(false));

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject(new Boolean(true));
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 12
@Test
public void testCategoricalVariable() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("dayofweek");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val("a:0.1", "b#dayofweek");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("1.0");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("1");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("0");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 13
public void testInsufficientLabelColumn() throws HiveException {
    BinarizeLabelUDTF udtf = new BinarizeLabelUDTF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    argOIs[0] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    List<String> featureNames = Arrays.asList("positive", "features");
    argOIs[1] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);

    udtf.initialize(argOIs);
}
 
Example 14
@Test
public void testEvaluateUserDictArray() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // userDictArray (from https://raw.githubusercontent.com/atilika/kuromoji/909fd6b32bf4e9dc86b7599de5c9b50ca8f004a1/kuromoji-core/src/test/resources/userdict.txt)
    List<String> userDict = new ArrayList<String>();
    userDict.add("日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞");
    userDict.add("関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,テスト名詞");
    argOIs[4] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, userDict);
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("日本経済新聞。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };

    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);

    Assert.assertNotNull(tokens);
    Assert.assertEquals(3, tokens.size());

    udf.close();
}
 
Example 15
@Test
public void testEvaluateUserDictURL() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector, null);
    // userDictUrl (Kuromoji official sample user defined dict on GitHub)
    // e.g., "日本経済新聞" will be "日本", "経済", and "新聞"
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text(
            "https://raw.githubusercontent.com/atilika/kuromoji/909fd6b32bf4e9dc86b7599de5c9b50ca8f004a1/kuromoji-core/src/test/resources/userdict.txt"));
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。日本経済新聞。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };

    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);

    Assert.assertNotNull(tokens);
    Assert.assertEquals(7, tokens.size());

    udf.close();
}