org.apache.hadoop.hive.serde2.io.DoubleWritable Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.io.DoubleWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexRSerde.java    From indexr with Apache License 2.0 6 votes vote down vote up
private static Writable createPrimitive(Object obj, PrimitiveObjectInspector inspector)
        throws SerDeException {
    if (obj == null) {
        return null;
    }
    switch (inspector.getPrimitiveCategory()) {
        case DOUBLE:
            return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
        case FLOAT:
            return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
        case INT:
            return new IntWritable(((IntObjectInspector) inspector).get(obj));
        case LONG:
            return new LongWritable(((LongObjectInspector) inspector).get(obj));
        case STRING:
            return new Text(((StringObjectInspector) inspector).getPrimitiveJavaObject(obj));
        case DATE:
            return ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        case TIMESTAMP:
            return ((TimestampObjectInspector) inspector).getPrimitiveWritableObject(obj);
        default:
            throw new SerDeException("Can't serialize primitive : " + inspector.getPrimitiveCategory());
    }
}
 
Example #2
Source File: TreePredictUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
public DoubleWritable evaluate(@Nonnull final String modelId, @Nonnull final Text script,
        @Nonnull final Vector features) throws HiveException {
    if (!modelId.equals(prevModelId)) {
        this.prevModelId = modelId;
        int length = script.getLength();
        byte[] b = script.getBytes();
        b = Base91.decode(b, 0, length);
        this.rNode = RegressionTree.deserialize(b, b.length, true);
    }
    Preconditions.checkNotNull(rNode);

    double value = rNode.predict(features);
    result.set(value);
    return result;
}
 
Example #3
Source File: UDFMathCosineSimilarity.java    From hive-third-functions with Apache License 2.0 6 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
    Object leftMapObj = arguments[0].get();
    Object rightMapObj = arguments[1].get();

    if (leftMapObj == null || rightMapObj == null) {
        return null;
    }

    Map<?, ?> leftMap = leftMapOI.getMap(leftMapObj);
    Map<?, ?> rightMap = leftMapOI.getMap(rightMapObj);

    Double normLeftMap = mapL2Norm(leftMap);
    Double normRightMap = mapL2Norm(rightMap);

    if (normLeftMap == null || normRightMap == null) {
        return null;
    }

    double dotProduct = mapDotProduct(leftMap, rightMap);
    return new DoubleWritable(dotProduct / (normLeftMap * normRightMap));
}
 
Example #4
Source File: RandomForestEnsembleUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public Object terminate(AggregationBuffer agg) throws HiveException {
    RfAggregationBufferV2 buf = (RfAggregationBufferV2) agg;
    if (buf._k == -1) {
        return null;
    }

    double[] posteriori = buf._posteriori;
    int label = smile.math.Math.whichMax(posteriori);
    smile.math.Math.unitize1(posteriori);
    double proba = posteriori[label];

    Object[] result = new Object[3];
    result[0] = new IntWritable(label);
    result[1] = new DoubleWritable(proba);
    result[2] = WritableUtils.toWritableList(posteriori);
    return result;
}
 
Example #5
Source File: DataToDoubleSummarySketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void partial1ModeIntKeysDefaultParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { intInspector, doubleInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToDoubleSummarySketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new IntWritable(1), new DoubleWritable(1)});
    eval.iterate(state, new Object[] {new IntWritable(2), new DoubleWritable(1)});

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 2);
    Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
    Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
        BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
    Assert.assertFalse(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);
  }
}
 
Example #6
Source File: ST_Z.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		return null;
	}
	if (!ogcGeometry.is3D()) {
		LogUtils.Log_Not3D(LOG);
		return null;
	}

	switch(GeometryUtils.getType(geomref)) {
	case ST_POINT:
		OGCPoint pt = (OGCPoint)ogcGeometry;
		resultDouble.set(pt.Z());
		return resultDouble;
	default:
		LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.getType(geomref));
		return null;
	}
}
 
Example #7
Source File: TreePredictUDFv1Test.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static double evalPredict(RegressionTree tree, double[] x)
        throws HiveException, IOException {
    String opScript = tree.predictOpCodegen(StackMachine.SEP);
    debugPrint(opScript);

    TreePredictUDFv1 udf = new TreePredictUDFv1();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.javaIntObjectInspector,
                PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(ModelType.opscode.getId()), new DeferredJavaObject(opScript),
            new DeferredJavaObject(ArrayUtils.toList(x)), new DeferredJavaObject(false)};

    DoubleWritable result = (DoubleWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example #8
Source File: ST_MultiPoint.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
public BytesWritable evaluate(DoubleWritable ... xyPairs) throws UDFArgumentLengthException{

		if (xyPairs == null || xyPairs.length == 0 ||  xyPairs.length%2 != 0) {
			LogUtils.Log_VariableArgumentLengthXY(LOG);
			return null;
		}

		try {
			MultiPoint mPoint = new MultiPoint();

			for (int i=0;i<xyPairs.length;i+=2){
				mPoint.add(xyPairs[i].get(), xyPairs[i+1].get());
			}

			return GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(mPoint, null, true));
		} catch (Exception e) {
		    LogUtils.Log_InternalError(LOG, "ST_MultiPoint: " + e);
		    return null;
		}
	}
 
Example #9
Source File: ST_MaxZ.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}
	if (!ogcGeometry.is3D()) {
		LogUtils.Log_Not3D(LOG);
		return null;
	}

	resultDouble.set(ogcGeometry.MaxZ());
	return resultDouble;
}
 
Example #10
Source File: TileX2LonUDF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public DoubleWritable evaluate(DeferredObject[] arguments) throws HiveException {
    Object arg0 = arguments[0].get();
    Object arg1 = arguments[1].get();

    if (arg0 == null) {
        return null;
    }
    if (arg1 == null) {
        throw new UDFArgumentException("zoom level should not be null");
    }

    int x = PrimitiveObjectInspectorUtils.getInt(arg0, xOI);
    int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI);
    Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class);

    final double lon;
    try {
        lon = GeoSpatialUtils.tilex2lon(x, zoom);
    } catch (IllegalArgumentException ex) {
        throw new UDFArgumentException(ex);
    }

    result.set(lon);
    return result;
}
 
Example #11
Source File: NumericHistogram.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
/**
 * In preparation for a Hive merge() call, serializes the current histogram object into an
 * ArrayList of DoubleWritable objects. This list is deserialized and merged by the merge
 * method.
 *
 * @return An ArrayList of Hadoop DoubleWritable objects that represents the current histogram.
 * @see #merge
 */
public ArrayList<DoubleWritable> serialize() {
    ArrayList<DoubleWritable> result = new ArrayList<DoubleWritable>();

    // Return a single ArrayList where the first element is the number of bins bins,
    // and subsequent elements represent bins (x,y) pairs.
    result.add(new DoubleWritable(nbins));
    if (bins != null) {
        for (int i = 0; i < nusedbins; i++) {
            result.add(new DoubleWritable(bins.get(i).x));
            result.add(new DoubleWritable(bins.get(i).y));
        }
    }

    return result;
}
 
Example #12
Source File: TestStMultiPolygon.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestStart() throws Exception {  // #110
	ST_MultiPolygon stMultiPolygon = new ST_MultiPolygon();
	List<DoubleWritable> args = new ArrayList<DoubleWritable>(7);
	args.add(dw0); args.add(dw1);
	args.add(dw1); args.add(dw0);
	args.add(dw0); args.add(dw0);
	BytesWritable rslt = stMultiPolygon.evaluate(args);
	// Text gty = typer.evaluate(rslt);
	// assertEquals(expty, gty.toString());
	// BytesWritable bwpg = stGeomN.evaluate(rslt, 0);
	BytesWritable cmp = stMultiPolygon.evaluate(new Text("multipolygon(((0 1, 1 0, 0 0)))"));
	//DeferredObject[] args = {rslt, cmp};
	//assertTrue(stEquals.evaluate(args));
	assertEquals(0.5, stArea.evaluate(rslt).get(), 0);
}
 
Example #13
Source File: WritableUtils.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
public static List<DoubleWritable> toWritableList(@Nonnull final double[] src) {
    // workaround to avoid a bug in Kryo
    // https://issues.apache.org/jira/browse/HIVE-12551
    /*
    final DoubleWritable[] writables = new DoubleWritable[src.length];
    for (int i = 0; i < src.length; i++) {
        writables[i] = new DoubleWritable(src[i]);
    }
    return Arrays.asList(writables);
    */
    final List<DoubleWritable> list = new ArrayList<DoubleWritable>(src.length);
    for (int i = 0; i < src.length; i++) {
        list.add(new DoubleWritable(src[i]));
    }
    return list;
}
 
Example #14
Source File: WritableUtils.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
public static List<DoubleWritable> newDoubleList(final int size, final double defaultValue) {
    // workaround to avoid a bug in Kryo
    // https://issues.apache.org/jira/browse/HIVE-12551
    /*
    final DoubleWritable[] array = new DoubleWritable[size];
    for (int i = 0; i < size; i++) {
        array[i] = new DoubleWritable(defaultValue);
    }
    return Arrays.asList(array);
    */
    final List<DoubleWritable> list = new ArrayList<DoubleWritable>(size);
    for (int i = 0; i < size; i++) {
        list.add(new DoubleWritable(defaultValue));
    }
    return list;
}
 
Example #15
Source File: DataToArrayOfDoublesSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeDoubleKeysExplicitParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, doubleInspector, doubleInspector, intInspector, floatInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToArrayOfDoublesSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    ArrayOfDoublesState state = (ArrayOfDoublesState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new DoubleWritable(1), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});
    eval.iterate(state, new Object[] {new DoubleWritable(2), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    eval.reset(state);
    result = eval.terminate(state);
    Assert.assertNull(result);
  }
}
 
Example #16
Source File: AUCUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public Object terminatePartial(AggregationBuffer agg) throws HiveException {
    ClassificationAUCAggregationBuffer myAggr = (ClassificationAUCAggregationBuffer) agg;

    Object[] partialResult = new Object[11];
    partialResult[0] = new DoubleWritable(myAggr.indexScore);
    partialResult[1] = new DoubleWritable(myAggr.area);
    partialResult[2] = new LongWritable(myAggr.fp);
    partialResult[3] = new LongWritable(myAggr.tp);
    partialResult[4] = new LongWritable(myAggr.fpPrev);
    partialResult[5] = new LongWritable(myAggr.tpPrev);
    partialResult[6] = myAggr.areaPartialMap;
    partialResult[7] = myAggr.fpPartialMap;
    partialResult[8] = myAggr.tpPartialMap;
    partialResult[9] = myAggr.fpPrevPartialMap;
    partialResult[10] = myAggr.tpPrevPartialMap;

    return partialResult;
}
 
Example #17
Source File: DataWritableWriter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private void writePrimitive(final Writable value) {
  if (value == null) {
    return;
  }
  if (value instanceof DoubleWritable) {
    recordConsumer.addDouble(((DoubleWritable) value).get());
  } else if (value instanceof BooleanWritable) {
    recordConsumer.addBoolean(((BooleanWritable) value).get());
  } else if (value instanceof FloatWritable) {
    recordConsumer.addFloat(((FloatWritable) value).get());
  } else if (value instanceof IntWritable) {
    recordConsumer.addInteger(((IntWritable) value).get());
  } else if (value instanceof LongWritable) {
    recordConsumer.addLong(((LongWritable) value).get());
  } else if (value instanceof ShortWritable) {
    recordConsumer.addInteger(((ShortWritable) value).get());
  } else if (value instanceof ByteWritable) {
    recordConsumer.addInteger(((ByteWritable) value).get());
  } else if (value instanceof BigDecimalWritable) {
    throw new UnsupportedOperationException("BigDecimal writing not implemented");
  } else if (value instanceof BinaryWritable) {
    recordConsumer.addBinary(((BinaryWritable) value).getBinary());
  } else {
    throw new IllegalArgumentException("Unknown value type: " + value + " " + value.getClass());
  }
}
 
Example #18
Source File: HaversineDistanceUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testMiles() throws HiveException, IOException {
    HaversineDistanceUDF udf = new HaversineDistanceUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)});

    // Tokyo
    double lat1 = 35.6833d, lon1 = 139.7667d;
    // Osaka
    double lat2 = 34.6603d, lon2 = 135.5232d;

    DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1),
            new DeferredJavaObject(lon1), new DeferredJavaObject(lat2),
            new DeferredJavaObject(lon2), new DeferredJavaObject(true)});
    Assert.assertEquals(249.84d, result1.get(), 0.1d);

    udf.close();
}
 
Example #19
Source File: ST_MinZ.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}
	if (!ogcGeometry.is3D()) {
		LogUtils.Log_Not3D(LOG);
		return null;
	}

	resultDouble.set(ogcGeometry.MinZ());
	return resultDouble;
}
 
Example #20
Source File: TreePredictUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
private static double evalPredict(RegressionTree tree, double[] x)
        throws HiveException, IOException {
    byte[] b = tree.serialize(true);
    byte[] encoded = Base91.encode(b);
    Text model = new Text(encoded);

    TreePredictUDF udf = new TreePredictUDF();
    udf.initialize(
        new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
                PrimitiveObjectInspectorFactory.writableStringObjectInspector,
                ObjectInspectorFactory.getStandardListObjectInspector(
                    PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
                ObjectInspectorUtils.getConstantObjectInspector(
                    PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)});
    DeferredObject[] arguments = new DeferredObject[] {new DeferredJavaObject("model_id#1"),
            new DeferredJavaObject(model), new DeferredJavaObject(ArrayUtils.toList(x)),
            new DeferredJavaObject(false)};

    DoubleWritable result = (DoubleWritable) udf.evaluate(arguments);
    udf.close();
    return result.get();
}
 
Example #21
Source File: TestStGeometryType.java    From spatial-framework-for-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void TestStGeometryType() throws Exception {
	ST_GeometryType typer = new ST_GeometryType();
	ST_Point stPt = new ST_Point();
	ST_MultiPoint stMp = new ST_MultiPoint();
	ST_LineString stLn = new ST_LineString();
	ST_Polygon stPoly = new ST_Polygon();
	BytesWritable bwGeom = stPt.evaluate(new DoubleWritable(0),
										 new DoubleWritable(0));
	Text gty = typer.evaluate(bwGeom);
	assertEquals("ST_POINT", gty.toString());
	bwGeom = stPt.evaluate(new Text("point z (10.02 20.01 2)"));
	gty = typer.evaluate(bwGeom);
	assertEquals("ST_POINT", gty.toString());
	bwGeom = stLn.evaluate(new Text("linestring (10 10, 20 20)"));
	gty = typer.evaluate(bwGeom);
	assertEquals("ST_LINESTRING", gty.toString());
	bwGeom = stPoly.evaluate(new Text("polygon ((0 0, 0 10, 10 0, 0 0))"));
	gty = typer.evaluate(bwGeom);
	assertEquals("ST_POLYGON", gty.toString());
}
 
Example #22
Source File: DataToSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeDoubleValuesExplicitParameters() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, intConstantInspector, floatConstantInspector, longConstantInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    final long seed = 2;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new DoubleWritable(1), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});
    eval.iterate(state, new Object[] {new DoubleWritable(2), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
  }
}
 
Example #23
Source File: DataToArrayOfDoublesSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeCheckTrimmingToNominal() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { intInspector, doubleInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToArrayOfDoublesSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    ArrayOfDoublesState state = (ArrayOfDoublesState) eval.getNewAggregationBuffer();
    for (int i = 0; i < 10000; i++) {
      eval.iterate(state, new Object[] {new IntWritable(i), new DoubleWritable(1)});
    }

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
    Assert.assertEquals(resultSketch.getEstimate(), 10000.0, 10000 * 0.03);
    Assert.assertTrue(resultSketch.getRetainedEntries() <= 4096, "retained entries: " + resultSketch.getRetainedEntries());

    eval.reset(state);
    result = eval.terminate(state);
    Assert.assertNull(result);
  }
}
 
Example #24
Source File: LDAPredictUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    OnlineLDAPredictAggregationBuffer myAggr = (OnlineLDAPredictAggregationBuffer) agg;
    if (myAggr.wcList.size() == 0) {
        return null;
    }

    Object[] partialResult = new Object[5];
    partialResult[0] = myAggr.wcList;
    partialResult[1] = myAggr.lambdaMap;
    partialResult[2] = new IntWritable(myAggr.topics);
    partialResult[3] = new FloatWritable(myAggr.alpha);
    partialResult[4] = new DoubleWritable(myAggr.delta);

    return partialResult;
}
 
Example #25
Source File: QuantifiedFeaturesUDTF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    int size = argOIs.length;
    if (size < 2) {
        throw new UDFArgumentException(
            "quantified_features takes at least two arguments: " + size);
    }
    this.boolOI = HiveUtils.asBooleanOI(argOIs[0]);

    int outputSize = size - 1;
    this.doubleOIs = new PrimitiveObjectInspector[outputSize];
    this.identifiers = new Identifier[outputSize];
    this.columnValues = new DoubleWritable[outputSize];

    for (int i = 0; i < outputSize; i++) {
        ObjectInspector argOI = argOIs[i + 1];
        if (HiveUtils.isNumberOI(argOI)) {
            doubleOIs[i] = HiveUtils.asDoubleCompatibleOI(argOI);
        } else {
            identifiers[i] = new Identifier<String>();
        }
        columnValues[i] = new DoubleWritable(Double.NaN);
    }

    this.forwardObjs = null;

    List<String> fieldNames = new ArrayList<String>(outputSize);
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(outputSize);
    fieldNames.add("features");
    fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example #26
Source File: MeanSquaredErrorUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public boolean iterate(DoubleWritable predicted, DoubleWritable actual)
        throws HiveException {
    if (predicted == null || actual == null) {// skip
        return true;
    }
    if (partial == null) {
        this.partial = new PartialResult();
    }
    partial.iterate(predicted.get(), actual.get());
    return true;
}
 
Example #27
Source File: MAPUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    MAPAggregationBuffer myAggr = (MAPAggregationBuffer) agg;

    Object[] partialResult = new Object[2];
    partialResult[0] = new DoubleWritable(myAggr.sum);
    partialResult[1] = new LongWritable(myAggr.count);
    return partialResult;
}
 
Example #28
Source File: FMeasureUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    FMeasureAggregationBuffer myAggr = (FMeasureAggregationBuffer) agg;

    Object[] partialResult = new Object[5];
    partialResult[0] = new LongWritable(myAggr.tp);
    partialResult[1] = new LongWritable(myAggr.totalActual);
    partialResult[2] = new LongWritable(myAggr.totalPredicted);
    partialResult[3] = new DoubleWritable(myAggr.beta);
    partialResult[4] = myAggr.average;
    return partialResult;
}
 
Example #29
Source File: FMeasureUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public DoubleWritable terminate(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    FMeasureAggregationBuffer myAggr = (FMeasureAggregationBuffer) agg;
    double result = myAggr.get();
    return new DoubleWritable(result);
}
 
Example #30
Source File: ArraySumUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public List<DoubleWritable> terminate() {
    if (partial == null) {
        return null;
    }

    final int size = partial._size;
    final List<Double> sum = partial._sum;

    final DoubleWritable[] ary = new DoubleWritable[size];
    for (int i = 0; i < size; i++) {
        Double d = sum.get(i);
        ary[i] = new DoubleWritable(d.doubleValue());
    }
    return Arrays.asList(ary);
}