Java Code Examples for com.tdunning.math.stats.TDigest#quantile()

The following examples show how to use com.tdunning.math.stats.TDigest#quantile() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PercentileCounterTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testTDigest() {
    double compression = 100;
    double quantile = 0.5;

    PercentileCounter counter = new PercentileCounter(compression, quantile);
    TDigest tDigest = TDigest.createAvlTreeDigest(compression);

    Random random = new Random();
    int dataSize = 10000;
    List<Double> dataset = Lists.newArrayListWithCapacity(dataSize);
    for (int i = 0; i < dataSize; i++) {
        double d = random.nextDouble();
        counter.add(d);
        tDigest.add(d);
    }
    double actualResult = counter.getResultEstimate();

    Collections.sort(dataset);
    double expectedResult = tDigest.quantile(quantile);

    assertEquals(expectedResult, actualResult, 0);
}

Example 2

Source File: TDigestTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testBasic() {
    int times = 1;
    int compression = 100;
    for (int t = 0; t < times; t++) {
        TDigest tDigest = TDigest.createAvlTreeDigest(compression);
        Random random = new Random();
        int dataSize = 10000;
        List<Double> dataset = Lists.newArrayListWithCapacity(dataSize);
        for (int i = 0; i < dataSize; i++) {
            double d = random.nextDouble();
            tDigest.add(d);
            dataset.add(d);
        }
        Collections.sort(dataset);

        double actualResult = tDigest.quantile(0.5);
        double expectedResult = MathUtil.findMedianInSortedList(dataset);
        assertEquals(expectedResult, actualResult, 0.01);
    }
}

Example 3

Source File: PercentileCounterTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testTDigest() {
    double compression = 100;
    double quantile = 0.5;

    PercentileCounter counter = new PercentileCounter(compression, quantile);
    TDigest tDigest = TDigest.createAvlTreeDigest(compression);

    Random random = new Random();
    int dataSize = 10000;
    List<Double> dataset = Lists.newArrayListWithCapacity(dataSize);
    for (int i = 0; i < dataSize; i++) {
        double d = random.nextDouble();
        counter.add(d);
        tDigest.add(d);
    }
    double actualResult = counter.getResultEstimate();

    Collections.sort(dataset);
    double expectedResult = tDigest.quantile(quantile);

    assertEquals(expectedResult, actualResult, 0);
}

Example 4

Source File: TDigestTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testBasic() {
    int times = 1;
    int compression = 100;
    for (int t = 0; t < times; t++) {
        TDigest tDigest = TDigest.createAvlTreeDigest(compression);
        Random random = new Random();
        int dataSize = 10000;
        List<Double> dataset = Lists.newArrayListWithCapacity(dataSize);
        for (int i = 0; i < dataSize; i++) {
            double d = random.nextDouble();
            tDigest.add(d);
            dataset.add(d);
        }
        Collections.sort(dataset);

        double actualResult = tDigest.quantile(0.5);
        double expectedResult = MathUtil.findMedianInSortedList(dataset);
        assertEquals(expectedResult, actualResult, 0.01);
    }
}

Example 5

Source File: ComparisonTest.java From t-digest with Apache License 2.0

6 votes

private void compareQD(PrintWriter out, AbstractContinousDistribution gen, String tag, long scale) {
    for (double compression : new double[]{10, 20, 50, 100, 200, 500, 1000, 2000}) {
        QDigest qd = new QDigest(compression);
        TDigest dist = new MergingDigest(compression);
        double[] data = new double[100000];
        for (int i = 0; i < 100000; i++) {
            double x = gen.nextDouble();
            dist.add(x);
            qd.offer((long) (x * scale));
            data[i] = x;
        }
        dist.compress();
        Arrays.sort(data);

        for (double q : new double[]{1e-5, 1e-4, 0.001, 0.01, 0.1, 0.5, 0.9, 0.99, 0.999, 0.9999, 0.99999}) {
            double x1 = dist.quantile(q);
            double x2 = (double) qd.getQuantile(q) / scale;
            double e1 = Dist.cdf(x1, data) - q;
            double e2 = Dist.cdf(x2, data) - q;
            out.printf("%s,%.0f,%.8f,%.10g,%.10g,%d,%d\n", tag, compression, q, e1, e2, dist.smallByteSize(), QDigest.serialize(qd).length);
        }
    }
}

Example 6

Source File: ComparisonTest.java From t-digest with Apache License 2.0

5 votes

private void compareSQ(PrintWriter out, AbstractContinousDistribution gen, String tag) {
    double[] quantiles = {0.001, 0.01, 0.1, 0.2, 0.3, 0.5, 0.7, 0.8, 0.9, 0.99, 0.999};
    for (double compression : new double[]{10, 20, 50, 100, 200, 500, 1000, 2000}) {
        QuantileEstimator sq = new QuantileEstimator(1001);
        TDigest dist = new MergingDigest(compression);
        double[] data = new double[100000];
        for (int i = 0; i < 100000; i++) {
            double x = gen.nextDouble();
            dist.add(x);
            sq.add(x);
            data[i] = x;
        }
        dist.compress();
        Arrays.sort(data);

        List<Double> qz = sq.getQuantiles();
        for (double q : quantiles) {
            double x1 = dist.quantile(q);
            double x2 = qz.get((int) (q * 1000 + 0.5));
            double e1 = Dist.cdf(x1, data) - q;
            double e2 = Dist.cdf(x2, data) - q;
            out.printf("%s,%.0f,%.8f,%.10g,%.10g,%d,%d\n",
                    tag, compression, q, e1, e2, dist.smallByteSize(), sq.serializedSize());

        }
    }
}

Example 7

Source File: PercentileTDigestAggregationFunction.java From incubator-pinot with Apache License 2.0

4 votes

@Override
public Double extractFinalResult(TDigest intermediateResult) {
  return intermediateResult.quantile(_percentile / 100.0);
}