Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#javaDoubleObjectInspector()

The following examples show how to use org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory#javaDoubleObjectInspector() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoubleNaturalOrder() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector};

    final Double[] values = new Double[] {3.1d, -1.1d, 4.1d, 2.1d, 5.1d};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(5, res.size());
    Assert.assertEquals(-1.1d, res.get(0));
    Assert.assertEquals(2.1d, res.get(1));
    Assert.assertEquals(3.1d, res.get(2));
    Assert.assertEquals(4.1d, res.get(3));
    Assert.assertEquals(5.1d, res.get(4));
}
 
Example 2
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTailKWithKey() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k -2")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 3
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testNullOnly() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector};

    final String[] values = new String[] {null, null, null};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertNull(res);
}
 
Example 4
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testTopKWithKey() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-k 2")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 5
Source File: SingularSpectrumTransformTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static int detectTwitterData(@Nonnull final ScoreFunction scoreFunc,
        @Nonnull final double threshold) throws IOException, HiveException {
    Parameters params = new Parameters();
    params.set(scoreFunc);
    PrimitiveObjectInspector oi = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    SingularSpectrumTransform sst = new SingularSpectrumTransform(params, oi);
    double[] outScores = new double[1];

    BufferedReader reader = readFile("twitter.csv.gz");
    println("# time x change");
    String line;
    int i = 1, numChangepoints = 0;
    while ((line = reader.readLine()) != null) {
        double x = Double.parseDouble(line);
        sst.update(x, outScores);
        printf("%d %f %f%n", i, x, outScores[0]);
        if (outScores[0] > threshold) {
            numChangepoints++;
        }
        i++;
    }

    return numChangepoints;
}
 
Example 6
Source File: UDAFToOrderedMapTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testTopK() throws Exception {
    TopKOrderedMapEvaluator evaluator = new TopKOrderedMapEvaluator();
    TopKOrderedMapEvaluator.MapAggregationBuffer agg =
            (TopKOrderedMapEvaluator.MapAggregationBuffer) evaluator.getNewAggregationBuffer();

    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaIntObjectInspector};

    final double[] keys = new double[] {0.7, 0.5, 0.8};
    final String[] values = new String[] {"banana", "apple", "candy"};
    int size = 2;

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < keys.length; i++) {
        evaluator.iterate(agg, new Object[] {keys[i], values[i], size});
    }

    Map<Object, Object> res = evaluator.terminate(agg);
    Object[] sortedValues = res.values().toArray();

    Assert.assertEquals(size, sortedValues.length);
    Assert.assertEquals("candy", sortedValues[0]);
    Assert.assertEquals("banana", sortedValues[1]);

    evaluator.close();
}
 
Example 7
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testNaturalOrderWithKey() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.7};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(3, res.size());
    Assert.assertEquals("apple", res.get(0));
    if (res.get(1) == "banana") { // duplicated key (0.7)
        Assert.assertEquals("candy", res.get(2));
    } else {
        Assert.assertEquals("banana", res.get(2));
    }
}
 
Example 8
Source File: ChangeFinder2DTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testCf1d() throws IOException, HiveException {
    Parameters params = new Parameters();
    params.set(LossFunction.logloss);
    PrimitiveObjectInspector oi = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    ListObjectInspector listOI = ObjectInspectorFactory.getStandardListObjectInspector(oi);
    ChangeFinder2D cf = new ChangeFinder2D(params, listOI);
    double[] outScores = new double[2];
    List<Double> x = new ArrayList<Double>(1);

    BufferedReader reader = readFile("cf1d.csv.gz");
    println("x outlier change");
    String line;
    int i = 1, numOutliers = 0, numChangepoints = 0;
    while ((line = reader.readLine()) != null) {
        double d = Double.parseDouble(line);
        x.add(Double.valueOf(d));

        cf.update(x, outScores);
        printf("%d %f %f %f%n", i, d, outScores[0], outScores[1]);
        if (outScores[0] > 10.d) {
            numOutliers++;
        }
        if (outScores[1] > 10.d) {
            numChangepoints++;
        }
        x.clear();
        i++;
    }
    Assert.assertTrue("#outliers SHOULD be greater than 10: " + numOutliers, numOutliers > 10);
    Assert.assertTrue("#outliers SHOULD be less than 20: " + numOutliers, numOutliers < 20);
    Assert.assertTrue("#changepoints SHOULD be greater than 0: " + numChangepoints,
        numChangepoints > 0);
    Assert.assertTrue("#changepoints SHOULD be less than 5: " + numChangepoints,
        numChangepoints < 5);
}
 
Example 9
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testReverseTailKWithKey() throws Exception {
    // = top-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k -2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("candy", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 10
Source File: VectorizeFeaturesUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testCategoricalVariable() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    udf.initialize(argOIs);

    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject("dayofweek");

    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    List<Text> expected = WritableUtils.val("a:0.1", "b#dayofweek");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("1.0");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("1");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);

    arguments[2] = new DeferredJavaObject("0");
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val(new String[] {"a:0.1"});
    Assert.assertEquals(expected, actuals);

    udf.close();
}
 
Example 11
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test(expected = UDFArgumentException.class)
public void testKVandVKFail() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -kv_map -vk_map")};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
}
 
Example 12
Source File: FeatureUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testTextDouble() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {
            new DeferredJavaObject(new Text("f1")), new DeferredJavaObject(2.5d)});

    Assert.assertEquals("f1:2.5", ret.toString());
}
 
Example 13
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testVKMapOptionBananaOverlap2() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "banana", "candy"};
    final double[] keys = new double[] {0.8, 0.8, 0.7};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(1, map.size());

    Assert.assertEquals(0.8d, map.get("banana"));
}
 
Example 14
Source File: FeatureUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntDouble() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1),
            new DeferredJavaObject(2.5d)});

    Assert.assertEquals("1:2.5", ret.toString());
}
 
Example 15
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testVKMapOption() throws Exception {
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -vk_map")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    Object result = evaluator.terminate(agg);

    Assert.assertEquals(LinkedHashMap.class, result.getClass());
    Map<?, ?> map = (Map<?, ?>) result;
    Assert.assertEquals(2, map.size());

    Assert.assertEquals(0.8d, map.get("candy"));
    Assert.assertEquals(0.7d, map.get("banana"));
}
 
Example 16
Source File: UDAFToOrderedListTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testReverseTopKWithKey() throws Exception {
    // = tail-k
    ObjectInspector[] inputOIs =
            new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                    ObjectInspectorUtils.getConstantObjectInspector(
                        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                        "-k 2 -reverse")};

    final String[] values = new String[] {"banana", "apple", "candy"};
    final double[] keys = new double[] {0.7, 0.5, 0.8};

    evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputOIs);
    evaluator.reset(agg);

    for (int i = 0; i < values.length; i++) {
        evaluator.iterate(agg, new Object[] {values[i], keys[i]});
    }

    @SuppressWarnings("unchecked")
    List<Object> res = (List<Object>) evaluator.terminate(agg);

    Assert.assertEquals(2, res.size());
    Assert.assertEquals("apple", res.get(0));
    Assert.assertEquals("banana", res.get(1));
}
 
Example 17
Source File: ArrayModelTest.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Test
public void testClassification() throws HiveException {
    final int ROW = 10, COL = 40;

    FactorizationMachineUDTF udtf = new FactorizationMachineUDTF();
    ListObjectInspector xOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    DoubleObjectInspector yOI = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    ObjectInspector paramOI = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        "-adareg -factors 20 -classification -seed 31 -iters 10 -int_feature -p " + COL);
    udtf.initialize(new ObjectInspector[] {xOI, yOI, paramOI});
    FactorizationMachineModel model = udtf.initModel(udtf._params);
    Assert.assertTrue("Actual class: " + model.getClass().getName(),
        model instanceof FMArrayModel);

    float accuracy = 0.f;
    final Random rnd = new Random(201L);
    for (int numberOfIteration = 0; numberOfIteration < 10000; numberOfIteration++) {
        ArrayList<IntFeature[]> fArrayList = new ArrayList<IntFeature[]>();
        ArrayList<Double> ans = new ArrayList<Double>();
        for (int i = 0; i < ROW; i++) {
            ArrayList<IntFeature> feature = new ArrayList<IntFeature>();
            for (int j = 1; j <= COL; j++) {
                if (i < (0.5f * ROW)) {
                    if (j == 1) {
                        feature.add(new IntFeature(j, 1.d));
                    } else if (j < 0.5 * COL) {
                        if (rnd.nextFloat() < 0.2f) {
                            feature.add(new IntFeature(j, rnd.nextDouble()));
                        }
                    }
                } else {
                    if (j > 0.5f * COL) {
                        if (rnd.nextFloat() < 0.2f) {
                            feature.add(new IntFeature(j, rnd.nextDouble()));
                        }
                    }
                }
            }
            IntFeature[] x = new IntFeature[feature.size()];
            feature.toArray(x);
            fArrayList.add(x);

            final double y;
            if (i < ROW * 0.5f) {
                y = -1.0d;
            } else {
                y = 1.0d;
            }
            ans.add(y);

            udtf.process(new Object[] {toStringArray(x), y});
        }
        int bingo = 0;
        int total = fArrayList.size();
        for (int i = 0; i < total; i++) {
            double tmpAns = ans.get(i);
            if (tmpAns < 0) {
                tmpAns = 0;
            } else {
                tmpAns = 1;
            }
            double p = model.predict(fArrayList.get(i));
            int predicted = p > 0.5 ? 1 : 0;
            if (predicted == tmpAns) {
                bingo++;
            }
        }
        accuracy = bingo / (float) total;
        println("Accuracy = " + accuracy);
    }
    udtf.runTrainingIteration(10);
    Assert.assertTrue("Expected accuracy greater than 0.95f: " + accuracy, accuracy > 0.95f);
}
 
Example 18
Source File: FactorizationMachineUDTFTest.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Test
public void testEarlyStopping() throws HiveException, IOException {
    println("Early stopping test");

    int iters = 20;

    // train with 20 iterations
    FactorizationMachineUDTF udtf = new FactorizationMachineUDTF();
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-factors 5 -min 1 -max 5 -init_v gaussian -eta0 0.002 -seed 31 -iters " + iters
                        + " -early_stopping -validation_threshold 1 -disable_cv")};

    udtf.initialize(argOIs);

    BufferedReader data = readFile("5107786.txt.gz");
    List<List<String>> featureVectors = new ArrayList<>();
    List<Double> ys = new ArrayList<>();
    String line = data.readLine();
    while (line != null) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");
        double y = Double.parseDouble(tokenizer.nextToken());
        List<String> features = new ArrayList<String>();
        while (tokenizer.hasMoreTokens()) {
            String f = tokenizer.nextToken();
            features.add(f);
        }
        udtf.process(new Object[] {features, y});
        featureVectors.add(features);
        ys.add(y);
        line = data.readLine();
    }
    udtf.finalizeTraining();
    data.close();

    double loss = udtf._validationState.getAverageLoss(featureVectors.size());
    Assert.assertTrue(
        "Training seems to be failed because average loss is greater than 0.1: " + loss,
        loss <= 0.1);

    Assert.assertNotNull("Early stopping validation has not been conducted",
        udtf._validationState);
    println("Performed " + udtf._validationState.getCurrentIteration() + " iterations out of "
            + iters);
    Assert.assertNotEquals("Early stopping did not happen", iters,
        udtf._validationState.getCurrentIteration());

    // store the best state achieved by early stopping
    iters = udtf._validationState.getCurrentIteration() - 2; // best loss was at (N-2)-th iter
    double cumulativeLoss = udtf._validationState.getCumulativeLoss();
    println("Cumulative loss: " + cumulativeLoss);

    // train with the number of early-stopped iterations
    udtf = new FactorizationMachineUDTF();
    argOIs[2] = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        "-factors 5 -min 1 -max 5 -init_v gaussian -eta0 0.002 -seed 31 -iters " + iters
                + " -early_stopping -validation_threshold 1 -disable_cv");
    udtf.initialize(argOIs);
    udtf.initModel(udtf._params);
    for (int i = 0, n = featureVectors.size(); i < n; i++) {
        udtf.process(new Object[] {featureVectors.get(i), ys.get(i)});
    }
    udtf.finalizeTraining();

    println("Performed " + udtf._validationState.getCurrentIteration() + " iterations out of "
            + iters);
    Assert.assertEquals("Training finished earlier than expected", iters,
        udtf._validationState.getCurrentIteration());

    println("Cumulative loss: " + udtf._validationState.getCumulativeLoss());
    Assert.assertTrue("Cumulative loss should be better than " + cumulativeLoss,
        cumulativeLoss > udtf._validationState.getCumulativeLoss());
}
 
Example 19
Source File: OrcUtils.java    From spork with Apache License 2.0 4 votes vote down vote up
public static ObjectInspector createObjectInspector(TypeInfo info) {
    switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.javaIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.javaLongObjectInspector;
        case STRING:
          return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
        case TIMESTAMP:
          return new PigJodaTimeStampObjectInspector();
        case DECIMAL:
          return new PigDecimalObjectInspector();
        case BINARY:
          return new PigDataByteArrayObjectInspector();
        case DATE:
        case VARCHAR:
        case BYTE:
        case SHORT:
            throw new IllegalArgumentException("Should never happen, " + 
                    (((PrimitiveTypeInfo) info).getPrimitiveCategory()) +
                    "is not valid Pig primitive data type");
        default:
            throw new IllegalArgumentException("Unknown primitive type " +
                    ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new PigStructInspector((StructTypeInfo) info);
    case MAP:
      return new PigMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new PigListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 
Example 20
Source File: HiveFactory.java    From transport with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@Override
public StdDouble createDouble(double value) {
  return new HiveDouble(value, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, this);
}