Java Code Examples for org.apache.hadoop.hive.ql.metadata.HiveException

The following examples show how to use org.apache.hadoop.hive.ql.metadata.HiveException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hive-funnel-udf   Source File: Funnel.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void iterate(AggregationBuffer aggregate, Object[] parameters) throws HiveException {
    FunnelAggregateBuffer funnelAggregate = (FunnelAggregateBuffer) aggregate;

    // Add the funnel steps if not already stored
    if (funnelAggregate.funnelSteps.isEmpty()) {
        // Funnel steps start at index 2
        addFunnelSteps(funnelAggregate, Arrays.copyOfRange(parameters, 2, parameters.length));
    }

    // Get the action_column value and add it (if it matches a funnel)
    Object action = parameters[0];
    Object timestamp = parameters[1];
    if (action != null && timestamp != null) {
        // Get the action value
        Object actionValue = ObjectInspectorUtils.copyToStandardObject(action, actionObjectInspector);
        // Get the timestamp value
        Object timestampValue = ObjectInspectorUtils.copyToStandardObject(timestamp, timestampObjectInspector);

        // If the action is not null and it is one of the funnels we are looking for, keep it
        if (actionValue != null && timestampValue != null && funnelAggregate.funnelSet.contains(actionValue)) {
            funnelAggregate.actions.add(actionValue);
            funnelAggregate.timestamps.add(timestampValue);
        }
    }
}
 
Example 2
Source Project: incubator-hivemall   Source File: MapIncludeKeysUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<?, ?> evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    if (arg0 == null) {
        return null;
    }
    final Map<?, ?> map = (Map<?, ?>) ObjectInspectorUtils.copyToStandardObject(arg0, mapOI,
        ObjectInspectorCopyOption.WRITABLE);

    Object arg1 = arguments[1].get();
    if (arg1 == null) {
        return null;
    }
    final List<?> filterKeys = (List<?>) ObjectInspectorUtils.copyToStandardObject(arg1, listOI,
        ObjectInspectorCopyOption.WRITABLE);

    final Map<Object, Object> result = new HashMap<>();
    for (Object k : filterKeys) {
        Object v = map.get(k);
        if (v != null) {
            result.put(k, v);
        }
    }
    return result;
}
 
Example 3
Source Project: incubator-hivemall   Source File: ArrayConcatUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<Object> evaluate(DeferredObject[] arguments) throws HiveException {
    ret.clear();

    for (int i = 0; i < arguments.length; i++) {
        final Object arrayObject = arguments[i].get();
        if (arrayObject == null) {
            continue;
        }

        final ListObjectInspector arrayOI = argumentOIs[i];
        final ObjectInspector elemOI = arrayOI.getListElementObjectInspector();
        final int arraylength = arrayOI.getListLength(arrayObject);
        for (int j = 0; j < arraylength; j++) {
            Object rawObj = arrayOI.getListElement(arrayObject, j);
            Object obj = ObjectInspectorUtils.copyToStandardObject(rawObj, elemOI,
                ObjectInspectorCopyOption.WRITABLE);
            ret.add(obj);
        }
    }

    return ret;
}
 
Example 4
@Test
public void testTruncate() throws HiveException {
    List<Text> args = WritableUtils.val("a:0.5", "b:1.0", "c:0.2");
    PolynomialFeaturesUDF udf = new PolynomialFeaturesUDF();
    List<Text> actuals = udf.evaluate(args, 3);
    List<Text> expected = WritableUtils.val("a:0.5", "a^a:0.25", "a^a^a:0.125", "a^a^c:0.05",
        "a^c:0.1", "a^c^c:0.020000001", "b:1.0", "c:0.2", "c^c:0.040000003", "c^c^c:0.008");
    Assert.assertEquals(expected, actuals);

    actuals = udf.evaluate(args, 3, false, false);
    expected = WritableUtils.val("a:0.5", "a^a:0.25", "a^a^a:0.125", "a^a^b:0.25", "a^a^c:0.05",
        "a^b:0.5", "a^b^b:0.5", "a^b^c:0.1", "a^c:0.1", "a^c^c:0.020000001", "b:1.0", "b^b:1.0",
        "b^b^b:1.0", "b^b^c:0.2", "b^c:0.2", "b^c^c:0.040000003", "c:0.2", "c^c:0.040000003",
        "c^c^c:0.008");
    Assert.assertEquals(expected, actuals);
}
 
Example 5
Source Project: incubator-hivemall   Source File: KPAPredictUDAF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
    super.init(m, parameters);

    // initialize input
    if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) {// from original data
        this.xhOI = HiveUtils.asNumberOI(parameters[0]);
        this.xkOI = HiveUtils.asNumberOI(parameters[1]);
        this.w0OI = HiveUtils.asNumberOI(parameters[2]);
        this.w1OI = HiveUtils.asNumberOI(parameters[3]);
        this.w2OI = HiveUtils.asNumberOI(parameters[4]);
        this.w3OI = HiveUtils.asNumberOI(parameters[5]);
    }

    return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
}
 
Example 6
Source Project: incubator-hivemall   Source File: ChangeFinderUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object[] evaluate(@Nonnull DeferredObject[] args) throws HiveException {
    Object x = args[0].get();
    if (x == null) {
        return _result;
    }

    _changeFinder.update(x, _scores);

    double outlierScore = _scores[0];
    double changepointScore = _scores[1];
    _outlierScore.set(outlierScore);
    _changepointScore.set(changepointScore);
    if (_isOutlier != null) {
        _isOutlier.set(outlierScore >= _params.outlierThreshold);
        if (_isChangepoint != null) {
            _isChangepoint.set(changepointScore >= _params.changepointThreshold);
        }
    }

    return _result;
}
 
Example 7
Source Project: incubator-hivemall   Source File: TreePredictUDFv1Test.java    License: Apache License 2.0 6 votes vote down vote up
private static int evalPredict(DecisionTree tree, double[] x)
        throws HiveException, IOException {
    String opScript = tree.predictOpCodegen(StackMachine.SEP);
    debugPrint(opScript);

    TreePredictUDFv1 udf = new TreePredictUDFv1();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(ModelType.opscode.getId()), new DeferredJavaObject(opScript),
            new DeferredJavaObject(ArrayUtils.toList(x)), new DeferredJavaObject(true)};

    IntWritable result = (IntWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example 8
Source Project: incubator-hivemall   Source File: TreePredictUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
public DoubleWritable evaluate(@Nonnull final String modelId, @Nonnull final Text script,
        @Nonnull final Vector features) throws HiveException {
    if (!modelId.equals(prevModelId)) {
        this.prevModelId = modelId;
        int length = script.getLength();
        byte[] b = script.getBytes();
        b = Base91.decode(b, 0, length);
        this.rNode = RegressionTree.deserialize(b, b.length, true);
    }
    Preconditions.checkNotNull(rNode);

    double value = rNode.predict(features);
    result.set(value);
    return result;
}
 
Example 9
@Test
public void testCosineSimilarity() throws HiveException {
    Double result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0), ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, 2 * 3 / (Math.sqrt(5) * Math.sqrt(10)), 0.0);
    result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0, "c", -1.0), ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, (2 * 3 + (-1) * 1) / (Math.sqrt(1 + 4 + 1) * Math.sqrt(1 + 9)), 0.0);
    result = getResult(ImmutableMap.<String, Double>of("a", 1.0, "b", 2.0, "c", -1.0), ImmutableMap.<String, Double>of("d", 1.0, "e", 3.0));
    assertEquals(result, 0.0, 0.0);
    result = getResult(null, ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, null);
    LinkedHashMap<String, Double> leftMap = Maps.newLinkedHashMap();
    leftMap.put("a", 1.0);
    leftMap.put("b", null);
    result = getResult(leftMap, ImmutableMap.<String, Double>of("c", 1.0, "b", 3.0));
    assertEquals(result, null);
}
 
Example 10
Source Project: incubator-hivemall   Source File: MapRouletteUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialization() throws HiveException, IOException {
    Map<String, Double> m = new HashMap<>();
    m.put("Tom", 0.1);
    m.put("Jerry", 0.2);
    m.put("Amy", 0.1);
    m.put("Wong", 0.1);
    m.put("Zhao", null);

    TestUtils.testGenericUDFSerialization(MapRouletteUDF.class,
        new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
            PrimitiveObjectInspectorFactory.javaStringObjectInspector,
            PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
        new Object[] {m});
    byte[] serialized = TestUtils.serializeObjectByKryo(new MapRouletteUDFTest());
    TestUtils.deserializeObjectByKryo(serialized, MapRouletteUDFTest.class);
}
 
Example 11
Source Project: incubator-hivemall   Source File: RescaleUDF.java    License: Apache License 2.0 6 votes vote down vote up
private static float min_max_normalization(final float value, final float min, final float max)
        throws HiveException {
    if (min > max) {
        throw new HiveException(
            "min value `" + min + "` SHOULD be less than max value `" + max + '`');
    }
    if (min == max) {
        return 0.5f;
    }
    if (value < min) {
        return 0.f;
    }
    if (value > max) {
        return 1.f;
    }
    return (value - min) / (max - min);
}
 
Example 12
private void flushBuffered(int position) throws HiveException {
    final Object[] forwardObjs = new Object[2];
    if (dense) {
        for (int i = 0; i < position; i++) {
            forwardObjs[0] = Float.valueOf(labels[i]);
            forwardObjs[1] = Arrays.asList(featuresFloatArray[i]);
            forward(forwardObjs);
        }
    } else {
        for (int i = 0; i < position; i++) {
            forwardObjs[0] = Float.valueOf(labels[i]);
            forwardObjs[1] = Arrays.asList(featuresArray[i]);
            forward(forwardObjs);
        }
    }
}
 
Example 13
Source Project: incubator-hivemall   Source File: MapRouletteUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSeed() throws HiveException, IOException {
    MapRouletteUDF udf = new MapRouletteUDF();
    Map<String, Double> m = new HashMap<>();
    udf.initialize(new ObjectInspector[] {
            ObjectInspectorFactory.getStandardMapObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaLongObjectInspector, 43L)});
    m.put("One", 0.7);
    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(m)};
    Assert.assertEquals("One", udf.evaluate(arguments));

    udf.close();
}
 
Example 14
Source Project: hive-funnel-udf   Source File: ConversionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertToConversionWithZeros() throws HiveException {
    Conversion udf = new Conversion();

    ObjectInspector[] inputOiList = new ObjectInspector[]{
            ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector)
    };

    udf.initialize(inputOiList);

    List<Long> inputList = Arrays.asList(10L, 5L, 0L, 0L, 0L);

    DeferredObject obj1 = mock(DeferredObject.class);
    DeferredObject[] objs = new DeferredObject[] { obj1 };
    when(obj1.get()).thenReturn(inputList);

    Assert.assertEquals(Arrays.asList(1.0, 0.5, 0.0, 0.0, 0.0), udf.evaluate(objs));
}
 
Example 15
Source Project: yauaa   Source File: TestParseUserAgent.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBasic() throws HiveException {
    // This is an edge case where the webview fields are calulcated AND wiped again.
    String userAgent = "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Version/4.0 Chrome/41.51020.2250.0246 Mobile Safari/537.36 cordova-amazon-fireos/3.4.0 AmazonWebAppPlatform/3.4.0;2.0";

    ParseUserAgent parseUserAgent = new ParseUserAgent();

    StandardStructObjectInspector resultInspector = (StandardStructObjectInspector) parseUserAgent
        .initialize(new ObjectInspector[]{
            PrimitiveObjectInspectorFactory.javaStringObjectInspector
        });

    for (int i = 0; i < 100000; i++) {
        Object row = parseUserAgent.evaluate(new DeferredObject[]{new DeferredJavaObject(userAgent)});
        checkField(resultInspector, row, "DeviceClass", "Tablet");
        checkField(resultInspector, row, "OperatingSystemNameVersion", "FireOS 3.4.0");
        checkField(resultInspector, row, "WebviewAppName", "Unknown");
    }
}
 
Example 16
Source Project: incubator-hivemall   Source File: LDAUDTFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRow() throws HiveException {
    LDAUDTF udtf = new LDAUDTF();
    final int numTopics = 2;
    ObjectInspector[] argOIs = new ObjectInspector[] {
            ObjectInspectorFactory.getStandardListObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector),
            ObjectInspectorUtils.getConstantObjectInspector(
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                "-topics " + numTopics)};
    udtf.initialize(argOIs);

    String[] doc1 = new String[] {"1", "2", "3"};
    udtf.process(new Object[] {Arrays.asList(doc1)});

    final MutableInt cnt = new MutableInt(0);
    udtf.setCollector(new Collector() {
        @Override
        public void collect(Object arg0) throws HiveException {
            cnt.addValue(1);
        }
    });
    udtf.close();

    Assert.assertEquals(doc1.length * numTopics, cnt.getValue());
}
 
Example 17
Source Project: incubator-hivemall   Source File: Lon2TileXUDF.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public IntWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    double lon = PrimitiveObjectInspectorUtils.getDouble(arg0, lonOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final int x;
    try {
        x = GeoSpatialUtils.lon2tilex(lon, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(x);
    return result;
}
 
Example 18
@Override
public void process(Object[] args) throws HiveException {
    if (mapToModel == null) {
        this.mapToModel = new HashMap<String, Predictor>();
    }
    if (args[1] == null) {// features is null
        return;
    }

    String modelId =
            PrimitiveObjectInspectorUtils.getString(nonNullArgument(args, 2), modelIdOI);
    Predictor model = mapToModel.get(modelId);
    if (model == null) {
        Text arg3 = modelOI.getPrimitiveWritableObject(nonNullArgument(args, 3));
        model = XGBoostUtils.loadPredictor(arg3);
        mapToModel.put(modelId, model);
    }

    Writable rowId = HiveUtils.copyToWritable(nonNullArgument(args, 0), rowIdOI);
    FVec features = denseFeatures ? parseDenseFeatures(args[1])
            : parseSparseFeatures(featureListOI.getList(args[1]));

    predictAndForward(model, rowId, features);
}
 
Example 19
Source Project: flink   Source File: HiveGenericUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object evalInternal(Object[] args) {

	for (int i = 0; i < args.length; i++) {
		((DeferredObjectAdapter) deferredObjects[i]).set(args[i]);
	}

	try {
		return HiveInspectors.toFlinkObject(returnInspector, function.evaluate(deferredObjects));
	} catch (HiveException e) {
		throw new FlinkHiveUDFException(e);
	}
}
 
Example 20
Source Project: incubator-hivemall   Source File: MapKeyValuesUDFTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStringDouble() throws HiveException, IOException {
    MapKeyValuesUDF udf = new MapKeyValuesUDF();

    udf.initialize(new ObjectInspector[] {ObjectInspectorFactory.getStandardMapObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector,
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)});

    Map<String, DoubleWritable> input = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        input.put("k" + i, new DoubleWritable(i));
    }

    GenericUDF.DeferredObject[] arguments =
            new GenericUDF.DeferredObject[] {new GenericUDF.DeferredJavaObject(input)};

    List<Object[]> actual = udf.evaluate(arguments);

    Assert.assertEquals(input.size(), actual.size());
    for (Object[] e : actual) {
        Assert.assertEquals(2, e.length);
        Object v = input.get(e[0]);
        Assert.assertEquals(e[1], v);
    }

    udf.close();
}
 
Example 21
Source Project: flink   Source File: TestHiveUDTF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void process(Object[] args) throws HiveException {
	String str = (String) args[0];
	for (String s : str.split(",")) {
		forward(s);
		forward(s);
	}
}
 
Example 22
Source Project: incubator-hivemall   Source File: XGBoostVersionUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
public String evaluate() throws HiveException {
    if (version == null) {
        this.version = XGBoostUtils.getVersion();
    }
    return version;
}
 
Example 23
Source Project: incubator-hivemall   Source File: CosineSimilarityUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FloatWritable evaluate(DeferredObject[] arguments) throws HiveException {
    List<String> ftvec1 = HiveUtils.asStringList(arguments[0], arg0ListOI);
    List<String> ftvec2 = HiveUtils.asStringList(arguments[1], arg1ListOI);
    float similarity = cosineSimilarity(ftvec1, ftvec2);
    return new FloatWritable(similarity);
}
 
Example 24
Source Project: incubator-hivemall   Source File: FMPredictGenericUDAF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    FMPredictAggregationBuffer buf = (FMPredictAggregationBuffer) agg;

    final Object[] partialResult = new Object[3];
    partialResult[0] = new DoubleWritable(buf.ret);
    if (buf.sumVjXj != null) {
        partialResult[1] = WritableUtils.toWritableList(buf.sumVjXj);
        partialResult[2] = WritableUtils.toWritableList(buf.sumV2X2);
    }
    return partialResult;
}
 
Example 25
@Test
public void testAdamHD() throws IOException, HiveException {
    String filePath = "adam_test_10000.tsv.gz";
    String options =
            "-loss logloss -opt AdamHD -reg l1 -lambda 0.0001 -iter 10 -mini_batch 1 -cv_rate 0.00005";

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();

    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector, params});

    BufferedReader reader = readFile(filePath);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");

        String featureLine = tokenizer.nextToken();
        List<String> X = Arrays.asList(featureLine.split(","));

        String labelLine = tokenizer.nextToken();
        Integer y = Integer.valueOf(labelLine);

        udtf.process(new Object[] {X, y});
    }

    udtf.finalizeTraining();

    Assert.assertTrue(
        "CumulativeLoss is expected to be less than 800: " + udtf.getCumulativeLoss(),
        udtf.getCumulativeLoss() < 800);
}
 
Example 26
@Override
protected Feature[] parseFeatures(@Nonnull final Object arg) throws HiveException {
    Feature[] features = Feature.parseFFMFeatures(arg, _xOI, _probes, _numFeatures, _numFields);
    if (_params.l2norm) {
        Feature.l2normalize(features);
    }
    return features;
}
 
Example 27
Source Project: incubator-hivemall   Source File: BitsCollectUDAF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void iterate(@SuppressWarnings("deprecation") AggregationBuffer aggr,
        Object[] parameters) throws HiveException {
    assert (parameters.length == 1);
    Object arg = parameters[0];
    if (arg != null) {
        int index = PrimitiveObjectInspectorUtils.getInt(arg, inputOI);
        if (index < 0) {
            throw new UDFArgumentException(
                "Specified index SHOULD NOT be negative: " + index);
        }
        ArrayAggregationBuffer agg = (ArrayAggregationBuffer) aggr;
        agg.bitset.set(index);
    }
}
 
Example 28
Source Project: hadoopcryptoledger   Source File: EthereumUDFTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
 public void EthereumGetSendAddressUDFNull() throws HiveException {
  EthereumGetSendAddressUDF egsaUDF = new EthereumGetSendAddressUDF();
ObjectInspector[] arguments = new ObjectInspector[2];
arguments[0] =  ObjectInspectorFactory.getReflectionObjectInspector(EthereumTransaction.class,ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
arguments[1] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
egsaUDF.initialize(arguments);	
assertNull(egsaUDF.evaluate(null),"Null argument to UDF returns null");
 }
 
Example 29
Source Project: incubator-hivemall   Source File: LastElementUDF.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
    Object list = args[0].get();
    if (list == null) {
        return null;
    }

    final int lastIdx = listInspector.getListLength(list) - 1;
    if (lastIdx >= 0) {
        return listInspector.getListElement(list, lastIdx);
    } else {
        return null;
    }
}
 
Example 30
@Test
public void testMomentum() throws IOException, HiveException {
    String filePath = "adam_test_10000.tsv.gz";
    String options =
            "-loss logloss -opt momentum -reg l1 -lambda 0.0001 -iter 10 -mini_batch 1 -cv_rate 0.00005";

    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();

    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, options);

    udtf.initialize(new ObjectInspector[] {stringListOI,
            PrimitiveObjectInspectorFactory.javaIntObjectInspector, params});

    BufferedReader reader = readFile(filePath);
    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        StringTokenizer tokenizer = new StringTokenizer(line, " ");

        String featureLine = tokenizer.nextToken();
        List<String> X = Arrays.asList(featureLine.split(","));

        String labelLine = tokenizer.nextToken();
        Integer y = Integer.valueOf(labelLine);

        udtf.process(new Object[] {X, y});
    }

    udtf.finalizeTraining();

    Assert.assertTrue(
        "CumulativeLoss is expected to be less than 1200: " + udtf.getCumulativeLoss(),
        udtf.getCumulativeLoss() < 1200);
}