org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: UDFStringSplitToMap.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_map(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_map, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 
Example #2
Source File: CoordinateCountryParseUDF.java    From occurrence with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 4) {
    throw new UDFArgumentException("parseCoordinates takes four arguments");
  }

  converters = new ObjectInspectorConverters.Converter[arguments.length];
  for (int i = 0; i < arguments.length; i++) {
    converters[i] = ObjectInspectorConverters
      .getConverter(arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
  }

  return ObjectInspectorFactory
    .getStandardStructObjectInspector(Arrays.asList("latitude", "longitude", "country"), Arrays
      .<ObjectInspector>asList(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));
}
 
Example #3
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #4
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateLongRow() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[1];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text(
                "商品の購入・詳細(サイズ、画像)は商品名をクリックしてください![L.B CANDY STOCK]フラワービジューベアドレス[L.B DAILY STOCK]ボーダーニットトップス[L.B DAILY STOCK]ボーダーロングニットOP[L.B DAILY STOCK]ロゴトートBAG[L.B DAILY STOCK]裏毛ロゴプリントプルオーバー【TVドラマ着用】アンゴラワッフルカーディガン【TVドラマ着用】グラフィティーバックリボンワンピース【TVドラマ着用】ボーダーハイネックトップス【TVドラマ着用】レオパードミッドカーフスカート【セットアップ対応商品】起毛ニットスカート【セットアップ対応商品】起毛ニットプルオーバー2wayサングラス33ナンバーリングニット3Dショルダーフレアードレス3周年スリッパ3周年ラグマット3周年ロックグラスキャンドルLily Brown 2015年 福袋MIXニットプルオーバーPeckhamロゴニットアンゴラジャガードプルオーバーアンゴラタートルアンゴラチュニックアンゴラニットカーディガンアンゴラニットプルオーバーアンゴラフレアワンピースアンゴラロングカーディガンアンゴラワッフルカーディガンヴィンテージファー付コートヴィンテージボーダーニットヴィンテージレースハイネックトップスヴィンテージレースブラウスウエストシースルーボーダーワンピースオーガンジーラインフレアスカートオープンショルダーニットトップスオフショルシャーリングワンピースオフショルニットオフショルニットプルオーバーオフショルボーダーロンパースオフショルワイドコンビネゾンオルテガ柄ニットプルオーバーカシュクールオフショルワンピースカットアシンメトリードレスカットサテンプリーツフレアースカートカラースーパーハイウェストスキニーカラーブロックドレスカラーブロックニットチュニックギャザーフレアスカートキラキラストライプタイトスカートキラキラストライプドレスキルティングファーコートグラデーションベアドレスグラデーションラウンドサングラスグラフティーオフショルトップスグラフティーキュロットグリッターリボンヘアゴムクロップドブラウスケーブルハイウエストスカートコーデュロイ×スエードパネルスカートコーデュロイタイトスカートゴールドバックルベルト付スカートゴシックヒールショートブーツゴシック柄ニットワンピコンビスタジャンサイドステッチボーイズデニムパンツサスペつきショートパンツサスペンダー付プリーツロングスカートシャーリングタイトスカートジャガードタックワンピーススエードフリルフラワーパンツスエード裏毛肩空きトップススクエアショルダーBAGスクエアバックルショルダースクエアミニバッグストーンビーチサンダルストライプサスペ付きスキニーストライプバックスリットシャツスライバーシャギーコートタートル×レースタイトスカートタートルニットプルオーバータイトジャンパースカートダブルクロスチュールフレアスカートダブルストラップパンプスダブルハートリングダブルフェイスチェックストールチェーンコンビビジューネックレスチェーンコンビビジューピアスチェーンコンビビジューブレスチェーンツバ広HATチェーンビジューピアスチェックニットプルオーバーチェックネルミディアムスカートチェック柄スキニーパンツチュールコンビアシメトップスデニムフレアースカートドットオフショルフリルブラウスドットジャガードドレスドットニットプルオーバードットレーストップスニット×オーガンジースカートセットニットキャミソールワンピースニットスヌードパールコンビフープピアスハイウエストショートデニムハイウエストタイトスカートハイウエストデニムショートパンツハイウエストプリーツスカートハイウエストミッドカーフスカートハイゲージタートルニットハイゲージラインニットハイネック切り替えスウェットバタフライネックレスバタフライミニピアスバタフライリングバックタンクリブワンピースバックリボンスキニーデニムパンツバックリボン深Vワンピースビジューストラップサンダルビスチェコンビオフショルブラウスブークレジャガードニットフェイクムートンショートコートフェレットカーディガンフェレットビックタートルニットブラウジングクルーブラウスプリーツブラウスフリルニットプルオーバーフリンジニットプルオーバーフレアニットスカートブロウ型サングラスベーシックフェレットプルオーバーベルト付ガウチョパンツベルト付ショートパンツベルト付タックスカートベルト付タックパンツベルベットインヒールパンプスベロアウェッジパンプスベロアミッドカーフワンピースベロアワンピースベロア風ニットカーディガンボア付コートボーダーVネックTシャツボーダーオフショルカットソーボーダーカットソーワンピースボーダータイトカットソーボーダートップスボーダートップス×スカートセットボストンメガネマオカラーシャツニットセットミックスニットプルオーバーミッドカーフ丈ポンチスカートミリタリーギャザーショートパンツメッシュハイネックトップスメルトンPコートメルトンダッフルコートメルトンダブルコートモヘアニットカーディガンモヘアニットタートルユリ柄プリーツフレアースカートライダースデニムジャケットライナー付チェスターコートラッフルプリーツブラウスラメジャガードハイゲージニットリブニットワンピリボン×パールバレッタリボンバレッタリボンベルトハイウエストパンツリリー刺繍開襟ブラウスレースビスチェローファーサボロゴニットキャップロゴ刺繍ニットワッチロングニットガウンワッフルアンゴラプルオーバーワンショルダワーワンピース光沢ラメニットカーディガン刺繍シフォンブラウス台形ミニスカート配色ニットプルオーバー裏毛プルオーバー×オーガンジースカートセット");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);
    Assert.assertNotNull(tokens);
    Assert.assertEquals(182, tokens.size());
    udf.close();
}
 
Example #5
Source File: VectorAddUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    this.xOI = HiveUtils.asListOI(argOIs[0]);
    this.yOI = HiveUtils.asListOI(argOIs[1]);
    this.xElemOI = HiveUtils.asNumberOI(xOI.getListElementObjectInspector());
    this.yElemOI = HiveUtils.asNumberOI(yOI.getListElementObjectInspector());

    if (HiveUtils.isIntegerOI(xElemOI) && HiveUtils.isIntegerOI(yElemOI)) {
        this.floatingPoints = false;
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaLongObjectInspector);
    } else {
        this.floatingPoints = true;
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    }
}
 
Example #6
Source File: FalloutTest.java    From hive-funnel-udf with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyFunnel() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList();

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(), udf.evaluate(objs));
}
 
Example #7
Source File: FFMPredictGenericUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.wiOI = HiveUtils.asDoubleCompatibleOI(parameters, 0);
        this.vijOI = HiveUtils.asListOI(parameters, 1);
        this.vijElemOI = HiveUtils.asFloatingPointOI(vijOI.getListElementObjectInspector());
        this.vjiOI = HiveUtils.asListOI(parameters, 2);
        this.vjiElemOI = HiveUtils.asFloatingPointOI(vjiOI.getListElementObjectInspector());
        this.xiOI = HiveUtils.asDoubleCompatibleOI(parameters, 3);
        this.xjOI = HiveUtils.asDoubleCompatibleOI(parameters, 4);
    } else {// from partial aggregation
        this.mergeInputOI = HiveUtils.asDoubleOI(parameters, 0);
    }

    return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
 
Example #8
Source File: VectorDotUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (!HiveUtils.isNumberListOI(argOI0)) {
        throw new UDFArgumentException(
            "Expected array<number> for the first argument: " + argOI0.getTypeName());
    }
    ListObjectInspector xListOI = HiveUtils.asListOI(argOI0);

    ObjectInspector argOI1 = argOIs[1];
    if (HiveUtils.isNumberListOI(argOI1)) {
        this.evaluator = new Dot2DVectors(xListOI, HiveUtils.asListOI(argOI1));
        return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    } else if (HiveUtils.isNumberOI(argOI1)) {
        this.evaluator = new Multiply2D1D(xListOI, argOI1);
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    } else {
        throw new UDFArgumentException(
            "Expected array<number> or number for the send argument: " + argOI1.getTypeName());
    }
}
 
Example #9
Source File: HaversineDistanceUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testMiles() throws HiveException, IOException {
    HaversineDistanceUDF udf = new HaversineDistanceUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});

    // Tokyo
    double lat1 = 35.6833d, lon1 = 139.7667d;
    // Osaka
    double lat2 = 34.6603d, lon2 = 135.5232d;

    DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1),
            new DeferredJavaObject(lon1), new DeferredJavaObject(lat2),
            new DeferredJavaObject(lon2), new DeferredJavaObject(true)});
    Assert.assertEquals(249.84d, result1.get(), 0.1d);

    udf.close();
}
 
Example #10
Source File: DoublesEvaluator.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  inputObjectInspector = (PrimitiveObjectInspector) parameters[0];

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are partial aggregations.
  if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
    if (parameters.length > 1) {
      kObjectInspector = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example #11
Source File: PassiveAggressiveUDTFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testPA1TrainWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);
}
 
Example #12
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTopK() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example #13
Source File: TestColumnStatistics.java    From hive-dwrf with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleStatisticsMerge() throws Exception {
  ObjectInspector inspector =
      PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;

  ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
  ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
  stats1.updateDouble(10.0);
  stats1.updateDouble(100.0);
  stats2.updateDouble(1.0);
  stats2.updateDouble(1000.0);
  stats1.merge(stats2);
  DoubleColumnStatistics typed = (DoubleColumnStatistics) stats1;
  assertEquals(1.0, typed.getMinimum(), 0.001);
  assertEquals(1000.0, typed.getMaximum(), 0.001);
}
 
Example #14
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTopKWithKey() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example #15
Source File: TryCastUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testList() throws IOException, HiveException {
    // try_cast(array(1.0,2.0,3.0), 'array<string>');
    TryCastUDF udf = new TryCastUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
                TypeInfoFactory.stringTypeInfo, new Text("array<string>"))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    Object result = udf.evaluate(args);

    Assert.assertEquals(WritableUtils.val("0.1", "1.1", "2.1"), result);

    udf.close();
}
 
Example #16
Source File: UDFJsonArrayExtractScalar.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
                "The function json_array_extract_scalar(json, json_path) takes exactly 2 arguments.");
    }

    converters = new ObjectInspectorConverters.Converter[arguments.length];
    for (int i = 0; i < arguments.length; i++) {
        converters[i] = ObjectInspectorConverters.getConverter(arguments[i],
                PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }

    return ObjectInspectorFactory
            .getStandardListObjectInspector(PrimitiveObjectInspectorFactory
                    .writableStringObjectInspector);
}
 
Example #17
Source File: TableMapping.java    From Hive-Cassandra with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize a object into bytes.
 * @param foi object inspector
 * @param decalred output object inspector
 * @param obj object to be serialized
 * @param useJsonSerialize true to use json serialization
 * @return object in serialized bytes
 * @throws IOException when error happens
 */
protected byte[] serializeToBytes(ObjectInspector foi, ObjectInspector doi, Object obj, boolean useJsonSerialize) throws IOException {
  serializeStream.reset();
  boolean isNotNull;
  if (!foi.getCategory().equals(Category.PRIMITIVE)
              && useJsonSerialize) {
    isNotNull = serialize(SerDeUtils.getJSONString(obj, foi),
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, doi, 1);
  } else {
    isNotNull = serialize(obj, foi, doi, 1);
  }
  if (!isNotNull) {
    return null;
  }
  byte[] key = new byte[serializeStream.getCount()];
  System.arraycopy(serializeStream.getData(), 0, key, 0, serializeStream.getCount());

  return key;
}
 
Example #18
Source File: GenerateSeriesUDTFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialization() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {}
    });

    udtf.process(new Object[] {1, new IntWritable(3)});

    byte[] serialized = TestUtils.serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, GenerateSeriesUDTF.class);
}
 
Example #19
Source File: FalloutTest.java    From hive-funnel-udf with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToFalloutWithZeros() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 0L, 0L, 0L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(0.0, 0.5, 0.0, 0.0, 0.0), udf.evaluate(objs));
}
 
Example #20
Source File: FMeasureUDAFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    fmeasure = new FMeasureUDAF();
    inputOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-beta 1.")};

    evaluator =
            fmeasure.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));

    agg = (FMeasureUDAF.FMeasureAggregationBuffer) evaluator.getNewAggregationBuffer();
}
 
Example #21
Source File: GenerateSeriesUDTFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoLongArgs() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector});

    final List<LongWritable> actual = new ArrayList<>();

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {
            Object[] row = (Object[]) args;
            LongWritable row0 = (LongWritable) row[0];
            actual.add(new LongWritable(row0.get()));
        }
    });

    udtf.process(new Object[] {1, new LongWritable(3)});

    List<LongWritable> expected =
            Arrays.asList(new LongWritable(1), new LongWritable(2), new LongWritable(3));
    Assert.assertEquals(expected, actual);
}
 
Example #22
Source File: BlurObjectInspectorGenerator.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
  switch (ti.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
  case STRUCT:
    StructTypeInfo sti = (StructTypeInfo) ti;
    List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
    for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
      ois.add(createObjectInspectorWorker(typeInfo));
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
  case LIST:
    ListTypeInfo lti = (ListTypeInfo) ti;
    TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
    return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
  default:
    throw new SerDeException("No Hive categories matched for [" + ti + "]");
  }
}
 
Example #23
Source File: TestColumnStatistics.java    From hive-dwrf with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringStatisticsMerge() throws Exception {
  ObjectInspector inspector =
      PrimitiveObjectInspectorFactory.javaStringObjectInspector;

  ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
  ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
  stats1.updateString("bob");
  stats1.updateString("david");
  stats1.updateString("charles");
  stats2.updateString("anne");
  stats2.updateString("erin");
  stats1.merge(stats2);
  StringColumnStatistics strStats = (StringColumnStatistics) stats1;
  assertEquals("anne", strStats.getMinimum());
  assertEquals("erin", strStats.getMaximum());
}
 
Example #24
Source File: MovingAverageUDTF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentException(
            "Two argument is expected for moving_avg(NUMBER value, const int windowSize): "
                    + argOIs.length);
    }
    this.valueOI = HiveUtils.asNumberOI(argOIs[0]);

    int windowSize = HiveUtils.getConstInt(argOIs[1]);
    this.movingAvg = new MovingAverage(windowSize);

    this.result = new DoubleWritable();
    this.forwardObjs = new Writable[] {result};

    List<String> fieldNames = Arrays.asList("avg");
    List<ObjectInspector> fieldOIs = Arrays.<ObjectInspector>asList(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #25
Source File: GenerateSeriesUDTFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testNegativeStepLong() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaLongObjectInspector,
                PrimitiveObjectInspectorFactory.writableIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    final List<LongWritable> actual = new ArrayList<>();

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {
            Object[] row = (Object[]) args;
            LongWritable row0 = (LongWritable) row[0];
            actual.add(new LongWritable(row0.get()));
        }
    });

    udtf.process(new Object[] {5L, new IntWritable(1), -2});

    List<LongWritable> expected =
            Arrays.asList(new LongWritable(5), new LongWritable(3), new LongWritable(1));
    Assert.assertEquals(expected, actual);
}
 
Example #26
Source File: UDFArrayIntersectTest.java    From hive-third-functions with Apache License 2.0 5 votes vote down vote up
@Test
public void testArrayIntersect() throws HiveException {
    UDFArrayIntersect udf = new UDFArrayIntersect();

    ObjectInspector leftArrayOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    ObjectInspector rightArrayOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    ObjectInspector[] arguments = {leftArrayOI, rightArrayOI};

    udf.initialize(arguments);

    assertTrue(Iterables.elementsEqual(ImmutableList.of(1,2,5), evaluate(ImmutableList.of(0,1,2,3,4,5), ImmutableList.of(1,1,2,2,5,5), udf)));
    assertTrue(Iterables.elementsEqual(ImmutableList.of(1,2,3,4), evaluate(ImmutableList.of(0,1,2,3,4,4), ImmutableList.of(1,1,2,2,3,4), udf)));
    assertTrue(Iterables.elementsEqual(ImmutableList.of(1,2,3,4), evaluate(ImmutableList.of(0,1,1,2,3,4,4), ImmutableList.of(1,1,2,2,3,4), udf)));
}
 
Example #27
Source File: SelectKBestUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] OIs) throws UDFArgumentException {
    if (OIs.length != 3) {
        throw new UDFArgumentLengthException("Specify three arguments: " + OIs.length);
    }

    if (!HiveUtils.isNumberListOI(OIs[0])) {
        throw new UDFArgumentTypeException(0,
            "Only array<number> type argument is acceptable but " + OIs[0].getTypeName()
                    + " was passed as `features`");
    }
    if (!HiveUtils.isNumberListOI(OIs[1])) {
        throw new UDFArgumentTypeException(1,
            "Only array<number> type argument is acceptable but " + OIs[1].getTypeName()
                    + " was passed as `importance_list`");
    }
    if (!HiveUtils.isIntegerOI(OIs[2])) {
        throw new UDFArgumentTypeException(2, "Only int type argument is acceptable but "
                + OIs[2].getTypeName() + " was passed as `k`");
    }

    this.featuresOI = HiveUtils.asListOI(OIs[0]);
    this.featureOI = HiveUtils.asDoubleCompatibleOI(featuresOI.getListElementObjectInspector());
    this.importanceListOI = HiveUtils.asListOI(OIs[1]);
    this.importanceElemOI =
            HiveUtils.asDoubleCompatibleOI(importanceListOI.getListElementObjectInspector());

    this._k = HiveUtils.getConstInt(OIs[2]);
    Preconditions.checkArgument(_k >= 1, UDFArgumentException.class);
    final List<DoubleWritable> result = new ArrayList<>(_k);
    for (int i = 0; i < _k; i++) {
        result.add(new DoubleWritable());
    }
    this._result = result;

    return ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
}
 
Example #28
Source File: FeatureUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntFloat() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1),
            new DeferredJavaObject(2.5f)});

    Assert.assertEquals("1:2.5", ret.toString());
}
 
Example #29
Source File: HitRateUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public void merge(@SuppressWarnings("deprecation") AggregationBuffer agg, Object partial)
        throws HiveException {
    if (partial == null) {
        return;
    }

    Object sumObj = internalMergeOI.getStructFieldData(partial, sumField);
    Object countObj = internalMergeOI.getStructFieldData(partial, countField);
    double sum = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(sumObj);
    long count = PrimitiveObjectInspectorFactory.writableLongObjectInspector.get(countObj);

    HitRateAggregationBuffer myAggr = (HitRateAggregationBuffer) agg;
    myAggr.merge(sum, count);
}
 
Example #30
Source File: FalloutTest.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Test(expected = UDFArgumentTypeException.class)
public void testBadInputType() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
        PrimitiveObjectInspectorFactory.javaLongObjectInspector
    };

    udf.initialize(inputOiList);
}