org.apache.hadoop.io.FloatWritable Java Examples

The following examples show how to use org.apache.hadoop.io.FloatWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataToSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeDoubleValuesExplicitParameters() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, intConstantInspector, floatConstantInspector, longConstantInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    final long seed = 2;
    UnionState state = (UnionState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new DoubleWritable(1), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});
    eval.iterate(state, new Object[] {new DoubleWritable(2), new IntWritable(16), new FloatWritable(0.99f), new LongWritable(seed)});

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
  }
}
 
Example #2
Source File: TestJoinTupleWritable.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void testIterable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable t = new TupleWritable(writs);
  for (int i = 0; i < 6; ++i) {
    t.setWritten(i);
  }
  verifIter(writs, t, 0);
}
 
Example #3
Source File: VirtualProgressReporter.java    From Cubert with Apache License 2.0 6 votes vote down vote up
/**
 * Set the progress of the current task.
 * *Note: Works only when using a Virtual Input Format
 *
 * @param value value of the progress must lie within [0.0, 1.0]
 */
public static void setProgress(float value)
{
    if (PhaseContext.isIntialized())
    {
        final MapContext mapContext = PhaseContext.getMapContext();
        try
        {
            final FloatWritable progress = (FloatWritable) mapContext.getCurrentKey();
            progress.set(value);
            mapContext.nextKeyValue();
        }
        catch (Exception e)
        {
            System.err.println("Unable to report progress in Load Cyclic. Exception: " + e);
            e.printStackTrace();
        }
    }
}
 
Example #4
Source File: NodeDumper.java    From nutch-htmlunit with Apache License 2.0 6 votes vote down vote up
/**
 * Outputs the host or domain as key for this record and numInlinks, numOutlinks
 * or score as the value.
 */
public void map(Text key, Node node,
  OutputCollector<Text, FloatWritable> output, Reporter reporter)
  throws IOException {

  float number = 0;
  if (inlinks) {
    number = node.getNumInlinks();
  }
  else if (outlinks) {
    number = node.getNumOutlinks();
  }
  else {
    number = node.getInlinkScore();
  }

  if (host) {
    key.set(URLUtil.getHost(key.toString()));
  } else {
    key.set(URLUtil.getDomainName(key.toString()));
  }

  output.collect(key, new FloatWritable(number));
}
 
Example #5
Source File: TestTupleWritable.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void testWritable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  sTuple.write(new DataOutputStream(out));
  ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
  TupleWritable dTuple = new TupleWritable();
  dTuple.readFields(new DataInputStream(in));
  assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
 
Example #6
Source File: CompactWritablesDeserializer.java    From Cubert with Apache License 2.0 6 votes vote down vote up
private static final WritableComparable<?> createWritable(DataType type)
{
    switch (type)
    {
    case BOOLEAN:
        return new BooleanWritable();
    case BYTE:
        return new ByteWritable();
    case INT:
        return new IntWritable();
    case LONG:
        return new LongWritable();
    case FLOAT:
        return new FloatWritable();
    case DOUBLE:
        return new DoubleWritable();
    case STRING:
        return new Text();
    default:
        return null;
    }
}
 
Example #7
Source File: NodeDumper.java    From anthelion with Apache License 2.0 6 votes vote down vote up
/**
 * Flips and collects the url and numeric sort value.
 */
public void reduce(FloatWritable key, Iterator<Text> values,
  OutputCollector<Text, FloatWritable> output, Reporter reporter)
  throws IOException {

  // take the negative of the negative to get original value, sometimes 0
  // value are a little weird
  float val = key.get();
  FloatWritable number = new FloatWritable(val == 0 ? 0 : -val);
  long numCollected = 0;

  // collect all values, this time with the url as key
  while (values.hasNext() && (numCollected < topn)) {
    Text url = WritableUtils.clone(values.next(), conf);
    output.collect(url, number);
    numCollected++;
  }
}
 
Example #8
Source File: PLSAPredictUDAF.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Override
public Object terminatePartial(@SuppressWarnings("deprecation") AggregationBuffer agg)
        throws HiveException {
    PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;
    if (myAggr.wcList.size() == 0) {
        return null;
    }

    Object[] partialResult = new Object[5];
    partialResult[0] = myAggr.wcList;
    partialResult[1] = myAggr.probMap;
    partialResult[2] = new IntWritable(myAggr.topics);
    partialResult[3] = new FloatWritable(myAggr.alpha);
    partialResult[4] = new DoubleWritable(myAggr.delta);

    return partialResult;
}
 
Example #9
Source File: DropoutNeuron.java    From incubator-retired-horn with Apache License 2.0 6 votes vote down vote up
@Override
public void backward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
    throws IOException {
  if (!this.isDropped()) {
    float delta = 0;

    for (Synapse<FloatWritable, FloatWritable> m : messages) {
      // Calculates error gradient for each neuron
      delta += (m.getDelta() * m.getWeight());

      // Weight corrections
      float weight = -this.getLearningRate() * m.getDelta()
          * this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
      this.push(weight);
    }

    this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
  } else {
    this.backpropagate(0);
  }
}
 
Example #10
Source File: DropoutNeuron.java    From incubator-retired-horn with Apache License 2.0 6 votes vote down vote up
@Override
public void forward(Iterable<Synapse<FloatWritable, FloatWritable>> messages)
    throws IOException {
  m2 = (isTraining()) ? MathUtils.getBinomial(1, 0.5) : 0.5f;

  if (m2 > 0) {
    float sum = 0;
    for (Synapse<FloatWritable, FloatWritable> m : messages) {
      sum += m.getInput() * m.getWeight();
    }

    this.setDrop(false);
    this.feedforward(squashingFunction.apply(sum) * m2);
  } else {
    this.setDrop(true);
    this.feedforward(0);
  }
}
 
Example #11
Source File: DataToDoubleSummarySketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void partial1ModeStringKeysExplicitParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, doubleInspector, intInspector, floatInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToDoubleSummarySketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new Text("a"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f)});
    eval.iterate(state, new Object[] {new Text("b"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f)});

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 2);
    Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
    Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
        BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
  }
}
 
Example #12
Source File: ExclusiveOrRecurrentMultiLayerPerceptron.java    From incubator-retired-horn with Apache License 2.0 6 votes vote down vote up
@Override
public void backward(
    Iterable<Synapse<FloatWritable, FloatWritable>> messages)
    throws IOException {
  float delta = 0;

  if (!this.isDropped()) {
    for (Synapse<FloatWritable, FloatWritable> m : messages) {
      // Calculates error gradient for each neuron
      delta += (m.getDelta() * m.getWeight());

      // Weight corrections
      float weight = -this.getLearningRate() * m.getDelta()
          * this.getOutput() + this.getMomentumWeight() * m.getPrevWeight();
      this.push(weight);
    }
  }

  this.backpropagate(delta * squashingFunction.applyDerivative(getOutput()));
}
 
Example #13
Source File: DataToArrayOfDoublesSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeDoubleKeysExplicitParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { doubleInspector, doubleInspector, doubleInspector, intInspector, floatInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToArrayOfDoublesSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkFinalResultInspector(resultInspector);

    ArrayOfDoublesState state = (ArrayOfDoublesState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new DoubleWritable(1), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});
    eval.iterate(state, new Object[] {new DoubleWritable(2), new DoubleWritable(1), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f)});

    Object result = eval.terminate(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof BytesWritable);
    ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);

    eval.reset(state);
    result = eval.terminate(state);
    Assert.assertNull(result);
  }
}
 
Example #14
Source File: TestTupleWritable.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private Writable[] makeRandomWritables() {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  return writs;
}
 
Example #15
Source File: TestTupleWritable.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testIterable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable t = new TupleWritable(writs);
  for (int i = 0; i < 6; ++i) {
    t.setWritten(i);
  }
  verifIter(writs, t, 0);
}
 
Example #16
Source File: TestTupleWritable.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testWritable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  sTuple.write(new DataOutputStream(out));
  ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
  TupleWritable dTuple = new TupleWritable();
  dTuple.readFields(new DataInputStream(in));
  assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
 
Example #17
Source File: TestJoinTupleWritable.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private Writable[] makeRandomWritables() {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  return writs;
}
 
Example #18
Source File: IndexRSerde.java    From indexr with Apache License 2.0 6 votes vote down vote up
private static Writable createPrimitive(Object obj, PrimitiveObjectInspector inspector)
        throws SerDeException {
    if (obj == null) {
        return null;
    }
    switch (inspector.getPrimitiveCategory()) {
        case DOUBLE:
            return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
        case FLOAT:
            return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
        case INT:
            return new IntWritable(((IntObjectInspector) inspector).get(obj));
        case LONG:
            return new LongWritable(((LongObjectInspector) inspector).get(obj));
        case STRING:
            return new Text(((StringObjectInspector) inspector).getPrimitiveJavaObject(obj));
        case DATE:
            return ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
        case TIMESTAMP:
            return ((TimestampObjectInspector) inspector).getPrimitiveWritableObject(obj);
        default:
            throw new SerDeException("Can't serialize primitive : " + inspector.getPrimitiveCategory());
    }
}
 
Example #19
Source File: TestJoinTupleWritable.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testWritable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  sTuple.write(new DataOutputStream(out));
  ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
  TupleWritable dTuple = new TupleWritable();
  dTuple.readFields(new DataInputStream(in));
  assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
 
Example #20
Source File: PageRankAlgorithm.java    From rheem with Apache License 2.0 6 votes vote down vote up
@Override
public Vertex<LongWritable, DoubleWritable, FloatWritable>
getCurrentVertex() throws IOException {
    Vertex<LongWritable, DoubleWritable, FloatWritable> vertex =
            getConf().createVertex();
    LongWritable vertexId = new LongWritable(
            (inputSplit.getSplitIndex() * totalRecords) + recordsRead);
    DoubleWritable vertexValue = new DoubleWritable(vertexId.get() * 10d);
    long targetVertexId =
            (vertexId.get() + 1) %
                    (inputSplit.getNumSplits() * totalRecords);
    float edgeValue = vertexId.get() * 100f;
    List<Edge<LongWritable, FloatWritable>> edges = Lists.newLinkedList();
    edges.add(EdgeFactory.create(new LongWritable(targetVertexId),
            new FloatWritable(edgeValue)));
    vertex.initialize(vertexId, vertexValue, edges);
    ++recordsRead;
    if (LOG.isInfoEnabled()) {
        LOG.info("next: Return vertexId=" + vertex.getId().get() +
                ", vertexValue=" + vertex.getValue() +
                ", targetVertexId=" + targetVertexId + ", edgeValue=" + edgeValue);
    }
    return vertex;
}
 
Example #21
Source File: DataToSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 6 votes vote down vote up
@Test
public void completeModeGivenK() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { floatInspector, intInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    checkResultInspector(resultInspector);

    SketchState state = (SketchState) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] { new FloatWritable(1), new IntWritable(400) });
    eval.iterate(state, new Object[] { new FloatWritable(2), new IntWritable(400) });

    BytesWritable bytes = (BytesWritable) eval.terminate(state);
    KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
    Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
    Assert.assertEquals(resultSketch.getNumRetained(), 2);
    Assert.assertEquals(resultSketch.getMinValue(), 1f);
    Assert.assertEquals(resultSketch.getMaxValue(), 2f);
  }
}
 
Example #22
Source File: TestObjectInspector.java    From hive-dwrf with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that after copying a lazy float object, calling materialize on the original and the
 * copy doesn't advance the tree reader twice
 * @throws Exception
 */
@Test
public void TestCopyFloat() throws Exception {
  ReaderWriterProfiler.setProfilerOptions(null);
  OrcLazyFloat lazyFloat = new OrcLazyFloat(new LazyFloatTreeReader(0, 0) {
    int nextCalls = 0;

    @Override
    public Object next(Object previous) throws IOException {
      if (nextCalls == 0) {
        return new FloatWritable(1.0f);
      }

      throw new IOException("next should only be called once");
    }

    @Override
    protected boolean seekToRow(long currentRow) throws IOException {
      return true;
    }
  });

  FloatObjectInspector floatOI = (FloatObjectInspector)
      OrcLazyObjectInspectorUtils.createLazyObjectInspector(TypeInfoFactory.floatTypeInfo);

  OrcLazyFloat lazyFloat2 = (OrcLazyFloat) floatOI.copyObject(lazyFloat);

  Assert.assertEquals(1.0f, ((FloatWritable) lazyFloat.materialize()).get());
  Assert.assertEquals(1.0f, ((FloatWritable) lazyFloat2.materialize()).get());
}
 
Example #23
Source File: DataToDoubleSummaryWithModeSketchUDAFTest.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
@Test
public void partial1ModeStringKeysExplicitParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { stringInspector, doubleInspector, intInspector, floatInspector, stringInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToDoubleSummaryWithModeSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new Text("a"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f), new Text("Min")});
    eval.iterate(state, new Object[] {new Text("b"), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f), new Text("Min")});
    eval.iterate(state, new Object[] {new Text("a"), new DoubleWritable(2), new IntWritable(32), new FloatWritable(0.99f), new Text("Min")});
    eval.iterate(state, new Object[] {new Text("b"), new DoubleWritable(1), new IntWritable(32), new FloatWritable(0.99f), new Text("Min")});

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 3);
    Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
    Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Min.toString());
    Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
        BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
    // because of sampling probability < 1
    Assert.assertTrue(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
    SketchIterator<DoubleSummary> it = resultSketch.iterator();
    while (it.next()) {
      Assert.assertEquals(it.getSummary().getValue(), 1.0);
    }
  }
}
 
Example #24
Source File: CosineDistanceUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public FloatWritable evaluate(DeferredObject[] arguments) throws HiveException {
    List<String> ftvec1 = HiveUtils.asStringList(arguments[0], arg0ListOI);
    List<String> ftvec2 = HiveUtils.asStringList(arguments[1], arg1ListOI);
    float d = 1.f - CosineSimilarityUDF.cosineSimilarity(ftvec1, ftvec2);
    return new FloatWritable(d);
}
 
Example #25
Source File: CosineSimilarityUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Override
public FloatWritable evaluate(DeferredObject[] arguments) throws HiveException {
    List<String> ftvec1 = HiveUtils.asStringList(arguments[0], arg0ListOI);
    List<String> ftvec2 = HiveUtils.asStringList(arguments[1], arg1ListOI);
    float similarity = cosineSimilarity(ftvec1, ftvec2);
    return new FloatWritable(similarity);
}
 
Example #26
Source File: JaccardIndexUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public FloatWritable evaluate(String a, String b, int k) {
    BigInteger ai = new BigInteger(a);
    BigInteger bi = new BigInteger(b);
    int countMatches = k - HammingDistanceUDF.hammingDistance(ai, bi);
    float jaccard = countMatches / (float) k;
    return val(2.f * (jaccard - 0.5f));
}
 
Example #27
Source File: BPRMFPredictionUDF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public FloatWritable evaluate(List<Float> Pu, List<Float> Qi, double Bi) throws HiveException {
    if (Pu == null && Qi == null) {
        return new FloatWritable(0.f);
    }
    if (Pu == null) {
        return new FloatWritable((float) Bi);
    } else if (Qi == null) {
        return new FloatWritable(0.f);
    }

    final int PuSize = Pu.size();
    final int QiSize = Qi.size();
    // workaround for TD        
    if (PuSize == 0) {
        if (QiSize == 0) {
            return new FloatWritable(0.f);
        } else {
            return new FloatWritable((float) Bi);
        }
    } else if (QiSize == 0) {
        return new FloatWritable(0.f);
    }

    if (QiSize != PuSize) {
        throw new HiveException("|Pu| " + PuSize + " was not equal to |Qi| " + QiSize);
    }

    float ret = (float) Bi;
    for (int k = 0; k < PuSize; k++) {
        ret += Pu.get(k) * Qi.get(k);
    }
    return new FloatWritable(ret);
}
 
Example #28
Source File: TypedBytesWritableOutput.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void write(Writable w) throws IOException {
  if (w instanceof TypedBytesWritable) {
    writeTypedBytes((TypedBytesWritable) w);
  } else if (w instanceof BytesWritable) {
    writeBytes((BytesWritable) w);
  } else if (w instanceof ByteWritable) {
    writeByte((ByteWritable) w);
  } else if (w instanceof BooleanWritable) {
    writeBoolean((BooleanWritable) w);
  } else if (w instanceof IntWritable) {
    writeInt((IntWritable) w);
  } else if (w instanceof VIntWritable) {
    writeVInt((VIntWritable) w);
  } else if (w instanceof LongWritable) {
    writeLong((LongWritable) w);
  } else if (w instanceof VLongWritable) {
    writeVLong((VLongWritable) w);
  } else if (w instanceof FloatWritable) {
    writeFloat((FloatWritable) w);
  } else if (w instanceof DoubleWritable) {
    writeDouble((DoubleWritable) w);
  } else if (w instanceof Text) {
    writeText((Text) w);
  } else if (w instanceof ArrayWritable) {
    writeArray((ArrayWritable) w);
  } else if (w instanceof MapWritable) {
    writeMap((MapWritable) w);
  } else if (w instanceof SortedMapWritable) {
    writeSortedMap((SortedMapWritable) w);
  } else if (w instanceof Record) {
    writeRecord((Record) w);
  } else {
    writeWritable(w); // last resort
  }
}
 
Example #29
Source File: ArgminKLDistanceUDAF.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public FloatWritable terminate() {
    if (partial == null) {
        return null;
    }
    if (partial.sum_inv_covar == 0.f) {// avoid null division
        return new FloatWritable(0.f);
    }
    float mean = (1.f / partial.sum_inv_covar) * partial.sum_mean_div_covar;
    return new FloatWritable(mean);
}
 
Example #30
Source File: WritableUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public static Writable toWritable(Object object) {
    if (object == null) {
        return null; //return NullWritable.get();
    }
    if (object instanceof Writable) {
        return (Writable) object;
    }
    if (object instanceof String) {
        return new Text((String) object);
    }
    if (object instanceof Long) {
        return new VLongWritable((Long) object);
    }
    if (object instanceof Integer) {
        return new VIntWritable((Integer) object);
    }
    if (object instanceof Byte) {
        return new ByteWritable((Byte) object);
    }
    if (object instanceof Double) {
        return new DoubleWritable((Double) object);
    }
    if (object instanceof Float) {
        return new FloatWritable((Float) object);
    }
    if (object instanceof Boolean) {
        return new BooleanWritable((Boolean) object);
    }
    if (object instanceof byte[]) {
        return new BytesWritable((byte[]) object);
    }
    return new BytesWritable(object.toString().getBytes());
}