org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FeatureHashingUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateListLibsvm() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            HiveUtils.getConstStringObjectInspector("-libsvm")});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES) + ":1", "0:1");
    Collections.sort(expected);
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}
 
Example #2
Source File: FalloutTest.java    From hive-funnel-udf with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToFalloutWithZeros() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 0L, 0L, 0L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(0.0, 0.5, 0.0, 0.0, 0.0), udf.evaluate(objs));
}
 
Example #3
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateOneRow() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[1];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);
    Assert.assertNotNull(tokens);
    Assert.assertEquals(5, tokens.size());
    udf.close();
}
 
Example #4
Source File: ConversionTest.java    From hive-funnel-udf with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyFunnel() throws HiveException {
    Conversion udf = new Conversion();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList();

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(), udf.evaluate(objs));
}
 
Example #5
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateLongRow() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[1];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text(
                "商品の購入・詳細(サイズ、画像)は商品名をクリックしてください![L.B CANDY STOCK]フラワービジューベアドレス[L.B DAILY STOCK]ボーダーニットトップス[L.B DAILY STOCK]ボーダーロングニットOP[L.B DAILY STOCK]ロゴトートBAG[L.B DAILY STOCK]裏毛ロゴプリントプルオーバー【TVドラマ着用】アンゴラワッフルカーディガン【TVドラマ着用】グラフィティーバックリボンワンピース【TVドラマ着用】ボーダーハイネックトップス【TVドラマ着用】レオパードミッドカーフスカート【セットアップ対応商品】起毛ニットスカート【セットアップ対応商品】起毛ニットプルオーバー2wayサングラス33ナンバーリングニット3Dショルダーフレアードレス3周年スリッパ3周年ラグマット3周年ロックグラスキャンドルLily Brown 2015年 福袋MIXニットプルオーバーPeckhamロゴニットアンゴラジャガードプルオーバーアンゴラタートルアンゴラチュニックアンゴラニットカーディガンアンゴラニットプルオーバーアンゴラフレアワンピースアンゴラロングカーディガンアンゴラワッフルカーディガンヴィンテージファー付コートヴィンテージボーダーニットヴィンテージレースハイネックトップスヴィンテージレースブラウスウエストシースルーボーダーワンピースオーガンジーラインフレアスカートオープンショルダーニットトップスオフショルシャーリングワンピースオフショルニットオフショルニットプルオーバーオフショルボーダーロンパースオフショルワイドコンビネゾンオルテガ柄ニットプルオーバーカシュクールオフショルワンピースカットアシンメトリードレスカットサテンプリーツフレアースカートカラースーパーハイウェストスキニーカラーブロックドレスカラーブロックニットチュニックギャザーフレアスカートキラキラストライプタイトスカートキラキラストライプドレスキルティングファーコートグラデーションベアドレスグラデーションラウンドサングラスグラフティーオフショルトップスグラフティーキュロットグリッターリボンヘアゴムクロップドブラウスケーブルハイウエストスカートコーデュロイ×スエードパネルスカートコーデュロイタイトスカートゴールドバックルベルト付スカートゴシックヒールショートブーツゴシック柄ニットワンピコンビスタジャンサイドステッチボーイズデニムパンツサスペつきショートパンツサスペンダー付プリーツロングスカートシャーリングタイトスカートジャガードタックワンピーススエードフリルフラワーパンツスエード裏毛肩空きトップススクエアショルダーBAGスクエアバックルショルダースクエアミニバッグストーンビーチサンダルストライプサスペ付きスキニーストライプバックスリットシャツスライバーシャギーコートタートル×レースタイトスカートタートルニットプルオーバータイトジャンパースカートダブルクロスチュールフレアスカートダブルストラップパンプスダブルハートリングダブルフェイスチェックストールチェーンコンビビジューネックレスチェーンコンビビジューピアスチェーンコンビビジューブレスチェーンツバ広HATチェーンビジューピアスチェックニットプルオーバーチェックネルミディアムスカートチェック柄スキニーパンツチュールコンビアシメトップスデニムフレアースカートドットオフショルフリルブラウスドットジャガードドレスドットニットプルオーバードットレーストップスニット×オーガンジースカートセットニットキャミソールワンピースニットスヌードパールコンビフープピアスハイウエストショートデニムハイウエストタイトスカートハイウエストデニムショートパンツハイウエストプリーツスカートハイウエストミッドカーフスカートハイゲージタートルニットハイゲージラインニットハイネック切り替えスウェットバタフライネックレスバタフライミニピアスバタフライリングバックタンクリブワンピースバックリボンスキニーデニムパンツバックリボン深Vワンピースビジューストラップサンダルビスチェコンビオフショルブラウスブークレジャガードニットフェイクムートンショートコートフェレットカーディガンフェレットビックタートルニットブラウジングクルーブラウスプリーツブラウスフリルニットプルオーバーフリンジニットプルオーバーフレアニットスカートブロウ型サングラスベーシックフェレットプルオーバーベルト付ガウチョパンツベルト付ショートパンツベルト付タックスカートベルト付タックパンツベルベットインヒールパンプスベロアウェッジパンプスベロアミッドカーフワンピースベロアワンピースベロア風ニットカーディガンボア付コートボーダーVネックTシャツボーダーオフショルカットソーボーダーカットソーワンピースボーダータイトカットソーボーダートップスボーダートップス×スカートセットボストンメガネマオカラーシャツニットセットミックスニットプルオーバーミッドカーフ丈ポンチスカートミリタリーギャザーショートパンツメッシュハイネックトップスメルトンPコートメルトンダッフルコートメルトンダブルコートモヘアニットカーディガンモヘアニットタートルユリ柄プリーツフレアースカートライダースデニムジャケットライナー付チェスターコートラッフルプリーツブラウスラメジャガードハイゲージニットリブニットワンピリボン×パールバレッタリボンバレッタリボンベルトハイウエストパンツリリー刺繍開襟ブラウスレースビスチェローファーサボロゴニットキャップロゴ刺繍ニットワッチロングニットガウンワッフルアンゴラプルオーバーワンショルダワーワンピース光沢ラメニットカーディガン刺繍シフォンブラウス台形ミニスカート配色ニットプルオーバー裏毛プルオーバー×オーガンジースカートセット");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);
    Assert.assertNotNull(tokens);
    Assert.assertEquals(182, tokens.size());
    udf.close();
}
 
Example #6
Source File: ConversionTest.java    From hive-funnel-udf with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToConversion() throws HiveException {
    Conversion udf = new Conversion();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 1L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(1.0, 0.5, 0.2), udf.evaluate(objs));
}
 
Example #7
Source File: ToJsonUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleArray() throws Exception {
    ToJsonUDF udf = new ToJsonUDF();

    ObjectInspector[] argOIs =
            new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)};
    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    udf.initialize(argOIs);
    Text serialized = udf.evaluate(args);

    Assert.assertEquals("[0.1,1.1,2.1]", serialized.toString());

    udf.close();
}
 
Example #8
Source File: HiveGeometryOIHelper.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
private OGCGeometry getGeometry(DeferredObject arg) {
	Object writable;
	try {
		writable = oi.getPrimitiveWritableObject(arg.get());
	} catch (HiveException e) {
		LOG.error("Failed to get writable", e);
		return null;
	}
	
	if (writable == null) {
		return null;
	}
	
	switch (oi.getPrimitiveCategory()) {
		case BINARY: return getGeometryFromBytes((BytesWritable)writable);
		case STRING: return OGCGeometry.fromText(((Text)writable).toString());
		default: return null;
	}
}
 
Example #9
Source File: FromJsonUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleArray() throws Exception {
    FromJsonUDF udf = new FromJsonUDF();

    String json = "[0.1,1.1,2.2]";
    String types = "array<double>";
    List<Double> expected = Arrays.asList(0.1d, 1.1d, 2.2d);

    ObjectInspector[] argOIs = new ObjectInspector[] {
            PrimitiveObjectInspectorFactory.writableStringObjectInspector,
            HiveUtils.getConstStringObjectInspector(types)};
    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(new Text(json)), null};

    udf.initialize(argOIs);
    Object result = udf.evaluate(args);

    Assert.assertEquals(expected, result);

    udf.close();
}
 
Example #10
Source File: TryCastUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testList() throws IOException, HiveException {
    // try_cast(array(1.0,2.0,3.0), 'array<string>');
    TryCastUDF udf = new TryCastUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
                TypeInfoFactory.stringTypeInfo, new Text("array<string>"))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    Object result = udf.evaluate(args);

    Assert.assertEquals(WritableUtils.val("0.1", "1.1", "2.1"), result);

    udf.close();
}
 
Example #11
Source File: TreePredictUDFv1Test.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static int evalPredict(DecisionTree tree, double[] x)
        throws HiveException, IOException {
    String opScript = tree.predictOpCodegen(StackMachine.SEP);
    debugPrint(opScript);

    TreePredictUDFv1 udf = new TreePredictUDFv1();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(ModelType.opscode.getId()), new DeferredJavaObject(opScript),
            new DeferredJavaObject(ArrayUtils.toList(x)), new DeferredJavaObject(true)};

    IntWritable result = (IntWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example #12
Source File: TreePredictUDFv1Test.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static double evalPredict(RegressionTree tree, double[] x)
        throws HiveException, IOException {
    String opScript = tree.predictOpCodegen(StackMachine.SEP);
    debugPrint(opScript);

    TreePredictUDFv1 udf = new TreePredictUDFv1();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(ModelType.opscode.getId()), new DeferredJavaObject(opScript),
            new DeferredJavaObject(ArrayUtils.toList(x)), new DeferredJavaObject(false)};

    DoubleWritable result = (DoubleWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example #13
Source File: TreePredictUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static int evalPredict(DecisionTree tree, double[] x)
        throws HiveException, IOException {
    byte[] b = tree.serialize(true);
    byte[] encoded = Base91.encode(b);
    Text model = new Text(encoded);

    TreePredictUDF udf = new TreePredictUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.writableStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(model), new DeferredJavaObject(ArrayUtils.toList(x)),
            new DeferredJavaObject(true)};

    Object[] result = (Object[]) udf.evaluate(arguments);
    udf.close();
    return ((IntWritable) result[0]).get();
}
 
Example #14
Source File: TreePredictUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static double evalPredict(RegressionTree tree, double[] x)
        throws HiveException, IOException {
    byte[] b = tree.serialize(true);
    byte[] encoded = Base91.encode(b);
    Text model = new Text(encoded);

    TreePredictUDF udf = new TreePredictUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.writableStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(model), new DeferredJavaObject(ArrayUtils.toList(x)),
            new DeferredJavaObject(false)};

    DoubleWritable result = (DoubleWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example #15
Source File: FeatureHashingUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateList() throws HiveException, IOException {
    FeatureHashingUDF udf = new FeatureHashingUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.val("apple:3", "orange:2", "banana", "0:1"))};

    List<String> expected = Arrays.asList(
        FeatureHashingUDF.mhash("apple", MurmurHash3.DEFAULT_NUM_FEATURES) + ":3",
        FeatureHashingUDF.mhash("orange", MurmurHash3.DEFAULT_NUM_FEATURES) + ":2",
        Integer.toString(FeatureHashingUDF.mhash("banana", MurmurHash3.DEFAULT_NUM_FEATURES)),
        "0:1");
    Assert.assertEquals(expected, udf.evaluate(args));

    udf.close();
}
 
Example #16
Source File: TestParseUserAgent.java    From yauaa with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasic() throws HiveException {
    // This is an edge case where the webview fields are calulcated AND wiped again.
    String userAgent = "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Version/4.0 Chrome/41.51020.2250.0246 Mobile Safari/537.36 cordova-amazon-fireos/3.4.0 AmazonWebAppPlatform/3.4.0;2.0";

    ParseUserAgent parseUserAgent = new ParseUserAgent();

    StandardStructObjectInspector resultInspector = (StandardStructObjectInspector) parseUserAgent
        .initialize(new ObjectInspector[]{
            PrimitiveObjectInspectorFactory.javaStringObjectInspector
        });

    for (int i = 0; i < 100000; i++) {
        Object row = parseUserAgent.evaluate(new DeferredObject[]{new DeferredJavaObject(userAgent)});
        checkField(resultInspector, row, "DeviceClass", "Tablet");
        checkField(resultInspector, row, "OperatingSystemNameVersion", "FireOS 3.4.0");
        checkField(resultInspector, row, "WebviewAppName", "Unknown");
    }
}
 
Example #17
Source File: VectorizeFeaturesUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testOneArgument() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    List<String> featureNames = Arrays.asList("a");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[2];
    arguments[1] = new DeferredJavaObject(new Double(0.1));

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example #18
Source File: VectorizeFeaturesUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoArguments() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("1.1");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val("a:0.1", "b:1.1");
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example #19
Source File: VectorizeFeaturesUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testAvoidZeroWeight() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("0");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example #20
Source File: ToStringArrayUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInputWithNullValue() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add(null);
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example #21
Source File: ToStringArrayUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTextArrayInput() throws HiveException, IOException {
    List<String> input = new ArrayList<String>(2);
    input.add("1");
    input.add("2");

    ToStringArrayUDF udf = new ToStringArrayUDF();
    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector)});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};
    List<String> output = udf.evaluate(args);

    Assert.assertEquals(input, output);

    udf.close();
}
 
Example #22
Source File: ToLibSVMFormatUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFeatureOnly() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 10")});

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(new DeferredObject[] {
            new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1"))}));

    Assert.assertEquals("3:2.1 7:3.4", udf.evaluate(
        new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1"))}));

    udf.close();
}
 
Example #23
Source File: ToLibSVMFormatUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFeatureAndIntLabel() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(
        new ObjectInspector[] {
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector),
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    "-features 10")});

    Assert.assertEquals("5 3:2.1 7:3.4",
        udf.evaluate(new DeferredObject[] {
                new DeferredJavaObject(Arrays.asList("apple:3.4", "orange:2.1")),
                new DeferredJavaObject(5)}));

    udf.close();
}
 
Example #24
Source File: ToLibSVMFormatUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFeatureAndFloatLabel() throws IOException, HiveException {
    ToLibSVMFormatUDF udf = new ToLibSVMFormatUDF();

    udf.initialize(
        new ObjectInspector[] {
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector),
                PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    "-features 10")});

    Assert.assertEquals("5.0 3:2.1 7:3.4",
        udf.evaluate(
            new DeferredObject[] {new DeferredJavaObject(Arrays.asList("7:3.4", "3:2.1")),
                    new DeferredJavaObject(5.f)}));

    udf.close();
}
 
Example #25
Source File: ArrayElementAtUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testString() throws IOException, HiveException {
    ArrayElementAtUDF udf = new ArrayElementAtUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(WritableUtils.val("s0", "s1", "s2")),
            new GenericUDF.DeferredJavaObject(1)};

    Assert.assertEquals(WritableUtils.val("s1"), udf.evaluate(args));

    udf.close();
}
 
Example #26
Source File: ArrayFlattenUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluate() throws HiveException, IOException {
    ArrayFlattenUDF udf = new ArrayFlattenUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaIntObjectInspector))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        Arrays.asList(Arrays.asList(0, 1, 2, 3), Arrays.asList(4, 5), Arrays.asList(6, 7)))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(8, result.size());
    for (int i = 0; i < 8; i++) {
        Assert.assertEquals(new IntWritable(i), result.get(i));
    }

    udf.close();
}
 
Example #27
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluate() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(new Double(3))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(4, result.size());
    for (int i = 0; i < 4; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example #28
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateAvoidNullAppend() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {
            new GenericUDF.DeferredJavaObject(
                WritableUtils.toWritableList(new double[] {0, 1, 2})),
            new GenericUDF.DeferredJavaObject(null)};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(3, result.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(new DoubleWritable(i), result.get(i));
    }

    udf.close();
}
 
Example #29
Source File: ArrayAppendUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateNullList() throws HiveException, IOException {
    ArrayAppendUDF udf = new ArrayAppendUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
            new GenericUDF.DeferredJavaObject(new Double(3d))};

    List<Object> result = udf.evaluate(args);

    Assert.assertEquals(Arrays.asList(new DoubleWritable(3d)), result);

    udf.close();
}
 
Example #30
Source File: ArrayToStrUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleCase() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaIntObjectInspector),
            PrimitiveObjectInspectorFactory.writableStringObjectInspector});

    Text sep = new Text("#");
    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
                    new GenericUDF.DeferredJavaObject(sep)};
    Assert.assertEquals("1#2#3", udf.evaluate(args));

    args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3)),
            new GenericUDF.DeferredJavaObject(null)};
    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}