Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-hivemall   Source File: FFMPredictGenericUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 5);
    super.init(mode, parameters);

    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.wiOI = HiveUtils.asDoubleCompatibleOI(parameters, 0);
        this.vijOI = HiveUtils.asListOI(parameters, 1);
        this.vijElemOI = HiveUtils.asFloatingPointOI(vijOI.getListElementObjectInspector());
        this.vjiOI = HiveUtils.asListOI(parameters, 2);
        this.vjiElemOI = HiveUtils.asFloatingPointOI(vjiOI.getListElementObjectInspector());
        this.xiOI = HiveUtils.asDoubleCompatibleOI(parameters, 3);
        this.xjOI = HiveUtils.asDoubleCompatibleOI(parameters, 4);
    } else {// from partial aggregation
        this.mergeInputOI = HiveUtils.asDoubleOI(parameters, 0);
    }

    return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
 
Example 2
Source Project: hive-funnel-udf   Source File: FalloutTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyFunnel() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList();

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(), udf.evaluate(objs));
}
 
Example 3
Source Project: incubator-hivemall   Source File: VectorAddUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    this.xOI = HiveUtils.asListOI(argOIs[0]);
    this.yOI = HiveUtils.asListOI(argOIs[1]);
    this.xElemOI = HiveUtils.asNumberOI(xOI.getListElementObjectInspector());
    this.yElemOI = HiveUtils.asNumberOI(yOI.getListElementObjectInspector());

    if (HiveUtils.isIntegerOI(xElemOI) && HiveUtils.isIntegerOI(yElemOI)) {
        this.floatingPoints = false;
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaLongObjectInspector);
    } else {
        this.floatingPoints = true;
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    }
}
 
Example 4
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvaluateLongRow() throws IOException, HiveException {
    KuromojiUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[1];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text(
                "商品の購入・詳細(サイズ、画像)は商品名をクリックしてください![L.B CANDY STOCK]フラワービジューベアドレス[L.B DAILY STOCK]ボーダーニットトップス[L.B DAILY STOCK]ボーダーロングニットOP[L.B DAILY STOCK]ロゴトートBAG[L.B DAILY STOCK]裏毛ロゴプリントプルオーバー【TVドラマ着用】アンゴラワッフルカーディガン【TVドラマ着用】グラフィティーバックリボンワンピース【TVドラマ着用】ボーダーハイネックトップス【TVドラマ着用】レオパードミッドカーフスカート【セットアップ対応商品】起毛ニットスカート【セットアップ対応商品】起毛ニットプルオーバー2wayサングラス33ナンバーリングニット3Dショルダーフレアードレス3周年スリッパ3周年ラグマット3周年ロックグラスキャンドルLily Brown 2015年 福袋MIXニットプルオーバーPeckhamロゴニットアンゴラジャガードプルオーバーアンゴラタートルアンゴラチュニックアンゴラニットカーディガンアンゴラニットプルオーバーアンゴラフレアワンピースアンゴラロングカーディガンアンゴラワッフルカーディガンヴィンテージファー付コートヴィンテージボーダーニットヴィンテージレースハイネックトップスヴィンテージレースブラウスウエストシースルーボーダーワンピースオーガンジーラインフレアスカートオープンショルダーニットトップスオフショルシャーリングワンピースオフショルニットオフショルニットプルオーバーオフショルボーダーロンパースオフショルワイドコンビネゾンオルテガ柄ニットプルオーバーカシュクールオフショルワンピースカットアシンメトリードレスカットサテンプリーツフレアースカートカラースーパーハイウェストスキニーカラーブロックドレスカラーブロックニットチュニックギャザーフレアスカートキラキラストライプタイトスカートキラキラストライプドレスキルティングファーコートグラデーションベアドレスグラデーションラウンドサングラスグラフティーオフショルトップスグラフティーキュロットグリッターリボンヘアゴムクロップドブラウスケーブルハイウエストスカートコーデュロイ×スエードパネルスカートコーデュロイタイトスカートゴールドバックルベルト付スカートゴシックヒールショートブーツゴシック柄ニットワンピコンビスタジャンサイドステッチボーイズデニムパンツサスペつきショートパンツサスペンダー付プリーツロングスカートシャーリングタイトスカートジャガードタックワンピーススエードフリルフラワーパンツスエード裏毛肩空きトップススクエアショルダーBAGスクエアバックルショルダースクエアミニバッグストーンビーチサンダルストライプサスペ付きスキニーストライプバックスリットシャツスライバーシャギーコートタートル×レースタイトスカートタートルニットプルオーバータイトジャンパースカートダブルクロスチュールフレアスカートダブルストラップパンプスダブルハートリングダブルフェイスチェックストールチェーンコンビビジューネックレスチェーンコンビビジューピアスチェーンコンビビジューブレスチェーンツバ広HATチェーンビジューピアスチェックニットプルオーバーチェックネルミディアムスカートチェック柄スキニーパンツチュールコンビアシメトップスデニムフレアースカートドットオフショルフリルブラウスドットジャガードドレスドットニットプルオーバードットレーストップスニット×オーガンジースカートセットニットキャミソールワンピースニットスヌードパールコンビフープピアスハイウエストショートデニムハイウエストタイトスカートハイウエストデニムショートパンツハイウエストプリーツスカートハイウエストミッドカーフスカートハイゲージタートルニットハイゲージラインニットハイネック切り替えスウェットバタフライネックレスバタフライミニピアスバタフライリングバックタンクリブワンピースバックリボンスキニーデニムパンツバックリボン深Vワンピースビジューストラップサンダルビスチェコンビオフショルブラウスブークレジャガードニットフェイクムートンショートコートフェレットカーディガンフェレットビックタートルニットブラウジングクルーブラウスプリーツブラウスフリルニットプルオーバーフリンジニットプルオーバーフレアニットスカートブロウ型サングラスベーシックフェレットプルオーバーベルト付ガウチョパンツベルト付ショートパンツベルト付タックスカートベルト付タックパンツベルベットインヒールパンプスベロアウェッジパンプスベロアミッドカーフワンピースベロアワンピースベロア風ニットカーディガンボア付コートボーダーVネックTシャツボーダーオフショルカットソーボーダーカットソーワンピースボーダータイトカットソーボーダートップスボーダートップス×スカートセットボストンメガネマオカラーシャツニットセットミックスニットプルオーバーミッドカーフ丈ポンチスカートミリタリーギャザーショートパンツメッシュハイネックトップスメルトンPコートメルトンダッフルコートメルトンダブルコートモヘアニットカーディガンモヘアニットタートルユリ柄プリーツフレアースカートライダースデニムジャケットライナー付チェスターコートラッフルプリーツブラウスラメジャガードハイゲージニットリブニットワンピリボン×パールバレッタリボンバレッタリボンベルトハイウエストパンツリリー刺繍開襟ブラウスレースビスチェローファーサボロゴニットキャップロゴ刺繍ニットワッチロングニットガウンワッフルアンゴラプルオーバーワンショルダワーワンピース光沢ラメニットカーディガン刺繍シフォンブラウス台形ミニスカート配色ニットプルオーバー裏毛プルオーバー×オーガンジースカートセット");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    @SuppressWarnings("unchecked")
    List<Text> tokens = (List<Text>) udf.evaluate(args);
    Assert.assertNotNull(tokens);
    Assert.assertEquals(182, tokens.size());
    udf.close();
}
 
Example 5
Source Project: incubator-hivemall   Source File: KuromojiUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example 6
@Test
public void testMiles() throws HiveException, IOException {
    HaversineDistanceUDF udf = new HaversineDistanceUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});

    // Tokyo
    double lat1 = 35.6833d, lon1 = 139.7667d;
    // Osaka
    double lat2 = 34.6603d, lon2 = 135.5232d;

    DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1),
            new DeferredJavaObject(lon1), new DeferredJavaObject(lat2),
            new DeferredJavaObject(lon2), new DeferredJavaObject(true)});
    Assert.assertEquals(249.84d, result1.get(), 0.1d);

    udf.close();
}
 
Example 7
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
  super.init(mode, parameters);
  inputObjectInspector = (PrimitiveObjectInspector) parameters[0];

  // Parameters:
  // In PARTIAL1 and COMPLETE mode, the parameters are original data.
  // In PARTIAL2 and FINAL mode, the parameters are partial aggregations.
  if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
    if (parameters.length > 1) {
      kObjectInspector = (PrimitiveObjectInspector) parameters[1];
    }
  }

  return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
}
 
Example 8
Source Project: incubator-hivemall   Source File: TryCastUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testList() throws IOException, HiveException {
    // try_cast(array(1.0,2.0,3.0), 'array<string>');
    TryCastUDF udf = new TryCastUDF();

    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
            PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
                TypeInfoFactory.stringTypeInfo, new Text("array<string>"))});

    DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(
        WritableUtils.toWritableList(new double[] {0.1, 1.1, 2.1}))};

    Object result = udf.evaluate(args);

    Assert.assertEquals(WritableUtils.val("0.1", "1.1", "2.1"), result);

    udf.close();
}
 
Example 9
@Test
public void testPA1TrainWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* define aggressive parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});

    /* train weights */
    List<?> features = (List<?>) intListOI.getList(new Object[] {1, 2, 3});
    udtf.train(features, 1);

    /* check weights */
    assertEquals(0.3333333f, udtf.model.get(1).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(2).get(), 1e-5f);
    assertEquals(0.3333333f, udtf.model.get(3).get(), 1e-5f);
}
 
Example 10
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTopK() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final String[] values = new String[] {"banana", "apple", "candy"};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 11
Source Project: hive-dwrf   Source File: TestColumnStatistics.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleStatisticsMerge() throws Exception {
  ObjectInspector inspector =
      PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;

  ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
  ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
  stats1.updateDouble(10.0);
  stats1.updateDouble(100.0);
  stats2.updateDouble(1.0);
  stats2.updateDouble(1000.0);
  stats1.merge(stats2);
  DoubleColumnStatistics typed = (DoubleColumnStatistics) stats1;
  assertEquals(1.0, typed.getMinimum(), 0.001);
  assertEquals(1000.0, typed.getMaximum(), 0.001);
}
 
Example 12
Source Project: incubator-hivemall   Source File: GenerateSeriesUDTFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNegativeStepLong() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaLongObjectInspector,
                PrimitiveObjectInspectorFactory.writableIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector});

    final List<LongWritable> actual = new ArrayList<>();

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {
            Object[] row = (Object[]) args;
            LongWritable row0 = (LongWritable) row[0];
            actual.add(new LongWritable(row0.get()));
        }
    });

    udtf.process(new Object[] {5L, new IntWritable(1), -2});

    List<LongWritable> expected =
            Arrays.asList(new LongWritable(5), new LongWritable(3), new LongWritable(1));
    Assert.assertEquals(expected, actual);
}
 
Example 13
Source Project: incubator-hivemall   Source File: UDAFToOrderedListTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTopKWithKey() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 14
Source Project: incubator-hivemall   Source File: MovingAverageUDTF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentException(
            "Two argument is expected for moving_avg(NUMBER value, const int windowSize): "
                    + argOIs.length);
    }
    this.valueOI = HiveUtils.asNumberOI(argOIs[0]);

    int windowSize = HiveUtils.getConstInt(argOIs[1]);
    this.movingAvg = new MovingAverage(windowSize);

    this.result = new DoubleWritable();
    this.forwardObjs = new Writable[] {result};

    List<String> fieldNames = Arrays.asList("avg");
    List<ObjectInspector> fieldOIs = Arrays.<ObjectInspector>asList(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 15
Source Project: hive-dwrf   Source File: TestColumnStatistics.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringStatisticsMerge() throws Exception {
  ObjectInspector inspector =
      PrimitiveObjectInspectorFactory.javaStringObjectInspector;

  ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(inspector);
  ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(inspector);
  stats1.updateString("bob");
  stats1.updateString("david");
  stats1.updateString("charles");
  stats2.updateString("anne");
  stats2.updateString("erin");
  stats1.merge(stats2);
  StringColumnStatistics strStats = (StringColumnStatistics) stats1;
  assertEquals("anne", strStats.getMinimum());
  assertEquals("erin", strStats.getMaximum());
}
 
Example 16
Source Project: incubator-hivemall   Source File: FMeasureUDAFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    fmeasure = new FMeasureUDAF();
    inputOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.writableLongObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-beta 1.")};

    evaluator =
            fmeasure.getEvaluator(new SimpleGenericUDAFParameterInfo(inputOIs, false, false));

    agg = (FMeasureUDAF.FMeasureAggregationBuffer) evaluator.getNewAggregationBuffer();
}
 
Example 17
Source Project: Hive-Cassandra   Source File: TableMapping.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Serialize a object into bytes.
 * @param foi object inspector
 * @param decalred output object inspector
 * @param obj object to be serialized
 * @param useJsonSerialize true to use json serialization
 * @return object in serialized bytes
 * @throws IOException when error happens
 */
protected byte[] serializeToBytes(ObjectInspector foi, ObjectInspector doi, Object obj, boolean useJsonSerialize) throws IOException {
  serializeStream.reset();
  boolean isNotNull;
  if (!foi.getCategory().equals(Category.PRIMITIVE)
              && useJsonSerialize) {
    isNotNull = serialize(SerDeUtils.getJSONString(obj, foi),
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, doi, 1);
  } else {
    isNotNull = serialize(obj, foi, doi, 1);
  }
  if (!isNotNull) {
    return null;
  }
  byte[] key = new byte[serializeStream.getCount()];
  System.arraycopy(serializeStream.getData(), 0, key, 0, serializeStream.getCount());

  return key;
}
 
Example 18
Source Project: hive-third-functions   Source File: UDFStringSplitToMap.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    // Check if two arguments were passed
    if (arguments.length != ARG_COUNT) {
        throw new UDFArgumentLengthException(
                "The function split_to_map(string, string, string) takes exactly " + ARG_COUNT + " arguments.");
    }

    // Check if two argument is of string
    for (int i = 0; i < 3; i++) {
        if (!ObjectInspectorUtils.compareTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, arguments[i])) {
            throw new UDFArgumentTypeException(i,
                    "\"" + PrimitiveObjectInspectorFactory.javaStringObjectInspector.getTypeName() + "\" "
                            + "expected at function split_to_map, but "
                            + "\"" + arguments[i].getTypeName() + "\" "
                            + "is found");
        }
    }

    ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector mapValueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;

    return ObjectInspectorFactory.getStandardMapObjectInspector(mapKeyOI, mapValueOI);
}
 
Example 19
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
        throw new UDFArgumentLengthException(
                "The function json_array_extract_scalar(json, json_path) takes exactly 2 arguments.");
    }

    converters = new ObjectInspectorConverters.Converter[arguments.length];
    for (int i = 0; i < arguments.length; i++) {
        converters[i] = ObjectInspectorConverters.getConverter(arguments[i],
                PrimitiveObjectInspectorFactory.writableStringObjectInspector);
    }

    return ObjectInspectorFactory
            .getStandardListObjectInspector(PrimitiveObjectInspectorFactory
                    .writableStringObjectInspector);
}
 
Example 20
Source Project: incubator-hivemall   Source File: GenerateSeriesUDTFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialization() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.writableIntObjectInspector});

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {}
    });

    udtf.process(new Object[] {1, new IntWritable(3)});

    byte[] serialized = TestUtils.serializeObjectByKryo(udtf);
    TestUtils.deserializeObjectByKryo(serialized, GenerateSeriesUDTF.class);
}
 
Example 21
Source Project: hive-funnel-udf   Source File: FalloutTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToFalloutWithZeros() throws HiveException {
    Fallout udf = new Fallout();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 0L, 0L, 0L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(0.0, 0.5, 0.0, 0.0, 0.0), udf.evaluate(objs));
}
 
Example 22
Source Project: incubator-hivemall   Source File: GenerateSeriesUDTFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoLongArgs() throws HiveException {
    GenerateSeriesUDTF udtf = new GenerateSeriesUDTF();

    udtf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.writableLongObjectInspector});

    final List<LongWritable> actual = new ArrayList<>();

    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object args) throws HiveException {
            Object[] row = (Object[]) args;
            LongWritable row0 = (LongWritable) row[0];
            actual.add(new LongWritable(row0.get()));
        }
    });

    udtf.process(new Object[] {1, new LongWritable(3)});

    List<LongWritable> expected =
            Arrays.asList(new LongWritable(1), new LongWritable(2), new LongWritable(3));
    Assert.assertEquals(expected, actual);
}
 
Example 23
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
  switch (ti.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
  case STRUCT:
    StructTypeInfo sti = (StructTypeInfo) ti;
    List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
    for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
      ois.add(createObjectInspectorWorker(typeInfo));
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
  case LIST:
    ListTypeInfo lti = (ListTypeInfo) ti;
    TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
    return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
  default:
    throw new SerDeException("No Hive categories matched for [" + ti + "]");
  }
}
 
Example 24
Source Project: occurrence   Source File: CoordinateCountryParseUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 4) {
    throw new UDFArgumentException("parseCoordinates takes four arguments");
  }

  converters = new ObjectInspectorConverters.Converter[arguments.length];
  for (int i = 0; i < arguments.length; i++) {
    converters[i] = ObjectInspectorConverters
      .getConverter(arguments[i], PrimitiveObjectInspectorFactory.writableStringObjectInspector);
  }

  return ObjectInspectorFactory
    .getStandardStructObjectInspector(Arrays.asList("latitude", "longitude", "country"), Arrays
      .<ObjectInspector>asList(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
        PrimitiveObjectInspectorFactory.javaStringObjectInspector));
}
 
Example 25
Source Project: incubator-hivemall   Source File: VectorDotUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if (argOIs.length != 2) {
        throw new UDFArgumentLengthException("Expected 2 arguments, but got " + argOIs.length);
    }

    ObjectInspector argOI0 = argOIs[0];
    if (!HiveUtils.isNumberListOI(argOI0)) {
        throw new UDFArgumentException(
            "Expected array<number> for the first argument: " + argOI0.getTypeName());
    }
    ListObjectInspector xListOI = HiveUtils.asListOI(argOI0);

    ObjectInspector argOI1 = argOIs[1];
    if (HiveUtils.isNumberListOI(argOI1)) {
        this.evaluator = new Dot2DVectors(xListOI, HiveUtils.asListOI(argOI1));
        return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    } else if (HiveUtils.isNumberOI(argOI1)) {
        this.evaluator = new Multiply2D1D(xListOI, argOI1);
        return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
    } else {
        throw new UDFArgumentException(
            "Expected array<number> or number for the send argument: " + argOI1.getTypeName());
    }
}
 
Example 26
Source Project: incubator-hivemall   Source File: ArrayToStrUDFTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNoSep() throws HiveException, IOException {
    ArrayToStrUDF udf = new ArrayToStrUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaIntObjectInspector)});

    DeferredObject[] args =
            new DeferredObject[] {new GenericUDF.DeferredJavaObject(Arrays.asList(1, 2, 3))};

    Assert.assertEquals("1,2,3", udf.evaluate(args));

    udf.close();
}
 
Example 27
Source Project: incubator-hivemall   Source File: SigmoidGenericUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector initialize(@Nonnull ObjectInspector[] argOIs)
        throws UDFArgumentException {
    if (argOIs.length != 1) {
        throw new UDFArgumentException("_FUNC_ takes 1 argument");
    }
    this.argOI = HiveUtils.asDoubleCompatibleOI(argOIs[0]);
    return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
}
 
Example 28
@Test(expected = UDFArgumentException.class)
public void testInvalidLossFunction() throws Exception {
    GeneralRegressorUDTF udtf = new GeneralRegressorUDTF();
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI =
            ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-loss HingeLoss");

    udtf.initialize(new ObjectInspector[] {stringListOI, floatOI, params});
}
 
Example 29
Source Project: incubator-hivemall   Source File: MapKeyValuesUDFTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStringDouble() throws HiveException, IOException {
    MapKeyValuesUDF udf = new MapKeyValuesUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    Map<String, DoubleWritable> input = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        input.put("k" + i, new DoubleWritable(i));
    }

    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};

    List<Object[]> actual = udf.evaluate(arguments);

    Assert.assertEquals(input.size(), actual.size());
    for (Object[] e : actual) {
        Assert.assertEquals(2, e.length);
        Object v = input.get(e[0]);
        Assert.assertEquals(e[1], v);
    }

    udf.close();
}
 
Example 30
Source Project: incubator-hivemall   Source File: ArrayAvgGenericUDAF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1);
    super.init(mode, parameters);
    // initialize input
    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputListOI = (ListObjectInspector) parameters[0];
        this.inputListElemOI =
                HiveUtils.asDoubleCompatibleOI(inputListOI.getListElementObjectInspector());
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.sizeField = soi.getStructFieldRef("size");
        this.sumField = soi.getStructFieldRef("sum");
        this.countField = soi.getStructFieldRef("count");
        this.sizeOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        this.sumOI = ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
        this.countOI = ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    }

    // initialize output
    final ObjectInspector outputOI;
    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {// terminate
        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    }
    return outputOI;
}