Java Code Examples for org.apache.commons.math3.stat.descriptive.SummaryStatistics#getStandardDeviation()

void doRandomTest(double errorPercentMax, double limitOfNotHitsPercent, int numPopulation) {
    final HttpServerExchange exchange = new HttpServerExchange(null);
    final Host[] newHosts = numPopulation < hosts.length ? Arrays.copyOf(hosts, numPopulation) : hosts;
    final Map<Integer, String> remains = IntStream.rangeClosed(0, newHosts.length - 1).boxed().collect(Collectors.toMap(x -> x, x -> ""));

    for (int retry = 1; retry <= NUM_RETRIES; retry++) {
        final SummaryStatistics statisticsOfResults = new SummaryStatistics();

        final Map<Integer, Integer> mapOfResults = new HashMap<>();
        new Random().ints(numPopulation).map(Math::abs).forEach(x -> {
            changeExchange(exchange, x);
            int result = getResult(exchange, newHosts);
            Integer lastCount = mapOfResults.get(result);
            remains.remove(result);
            mapOfResults.put(result, lastCount != null ? ++lastCount : 0);
        });
        mapOfResults.entrySet().stream().mapToDouble(Map.Entry::getValue).forEach(statisticsOfResults::addValue);
        double errorPercent = (statisticsOfResults.getStandardDeviation() / numPopulation) * 100;
        assertThat(errorPercent, lessThan(errorPercentMax));
    }
    final List<Integer> listOfNotHit = remains.entrySet().stream().map(Map.Entry::getKey).collect(toList());
    assertThat(listOfNotHit.size(), lessThanOrEqualTo((int) (newHosts.length * (limitOfNotHitsPercent / 100))));
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    final double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                   return getKernel(stats).sample();
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

private static Stats getStats(NumericColumn<?> values, SummaryStatistics summaryStatistics) {
  Stats stats = new Stats("Column: " + values.name());
  stats.min = summaryStatistics.getMin();
  stats.max = summaryStatistics.getMax();
  stats.n = summaryStatistics.getN();
  stats.sum = summaryStatistics.getSum();
  stats.variance = summaryStatistics.getVariance();
  stats.populationVariance = summaryStatistics.getPopulationVariance();
  stats.quadraticMean = summaryStatistics.getQuadraticMean();
  stats.geometricMean = summaryStatistics.getGeometricMean();
  stats.mean = summaryStatistics.getMean();
  stats.standardDeviation = summaryStatistics.getStandardDeviation();
  stats.sumOfLogs = summaryStatistics.getSumOfLogs();
  stats.sumOfSquares = summaryStatistics.getSumsq();
  stats.secondMoment = summaryStatistics.getSecondMoment();
  return stats;
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    final double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                   return getKernel(stats).sample();
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

private static Stats getStats(NumericColumn<?> values, SummaryStatistics summaryStatistics) {
  Stats stats = new Stats("Column: " + values.name());
  stats.min = summaryStatistics.getMin();
  stats.max = summaryStatistics.getMax();
  stats.n = summaryStatistics.getN();
  stats.sum = summaryStatistics.getSum();
  stats.variance = summaryStatistics.getVariance();
  stats.populationVariance = summaryStatistics.getPopulationVariance();
  stats.quadraticMean = summaryStatistics.getQuadraticMean();
  stats.geometricMean = summaryStatistics.getGeometricMean();
  stats.mean = summaryStatistics.getMean();
  stats.standardDeviation = summaryStatistics.getStandardDeviation();
  stats.sumOfLogs = summaryStatistics.getSumOfLogs();
  stats.sumOfSquares = summaryStatistics.getSumsq();
  stats.secondMoment = summaryStatistics.getSecondMoment();
  return stats;
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    final double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                   return getKernel(stats).sample();
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    final double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                   return randomData.nextGaussian(stats.getMean(),
                                                  stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
    try {
        randomData.nextGaussian(0, 0);
        Assert.fail("zero sigma -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
        // ignored
    }
    SummaryStatistics u = new SummaryStatistics();
    for (int i = 0; i < largeSampleSize; i++) {
        u.addValue(randomData.nextGaussian(0, 1));
    }
    double xbar = u.getMean();
    double s = u.getStandardDeviation();
    double n = u.getN();
    /*
     * t-test at .001-level TODO: replace with externalized t-test, with
     * test statistic defined in TestStatistic
     */
    Assert.assertTrue(FastMath.abs(xbar) / (s / FastMath.sqrt(n)) < 3.29);
}
 

/**
 * Generates a random value from this distribution.
 * <strong>Preconditions:</strong><ul>
 * <li>the distribution must be loaded before invoking this method</li></ul>
 * @return the random value.
 * @throws MathIllegalStateException if the distribution has not been loaded
 */
public double getNextValue() throws MathIllegalStateException {

    if (!loaded) {
        throw new MathIllegalStateException(LocalizedFormats.DISTRIBUTION_NOT_LOADED);
    }

    // Start with a uniformly distributed random number in (0,1)
    double x = randomData.nextUniform(0,1);

    // Use this to select the bin and generate a Gaussian within the bin
    for (int i = 0; i < binCount; i++) {
       if (x <= upperBounds[i]) {
           SummaryStatistics stats = binStats.get(i);
           if (stats.getN() > 0) {
               if (stats.getStandardDeviation() > 0) {  // more than one obs
                    return randomData.nextGaussian
                        (stats.getMean(),stats.getStandardDeviation());
               } else {
                   return stats.getMean(); // only one obs in bin
               }
           }
       }
    }
    throw new MathIllegalStateException(LocalizedFormats.NO_BIN_SELECTED);
}
 

public static double computePriceChangeSTD(double[] open, double[] close) {
	SummaryStatistics stats = new SummaryStatistics();
	for (int i = 0; i < open.length; i++) {
		stats.addValue(close[i] - open[i]);
	}
	return stats.getStandardDeviation();
}
 

/**
 * Initializes the model using a training set of anomaly scores.
 *
 * The hybrid model initialization has several steps. First, a log-normal
 * distribution is fit to the training set scores. Next, the quantile sketch
 * is initialized with at {@code numLogNormalQuantiles} samples from the
 * log-normal model up to {@code maxScore}.
 *
 * @param anomalyScores  an array of anomaly scores with which to train the model.
 */
@Override
public void train(double[] anomalyScores) {
    /*
      We assume the anomaly scores are fit to a log-normal distribution.
      Equivalent to fitting a Gaussian to the logs of the anomaly scores.
    */
    SummaryStatistics stats = new SummaryStatistics();
    for (int i = 0; i < anomalyScores.length; i++) {
        stats.addValue(Math.log(anomalyScores[i]));
    }
    final double mu = stats.getMean();
    final double sigma = stats.getStandardDeviation();

    /*
      Compute the 1/R quantiles for R = `numLogNormalQuantiles` of the
      corresponding log-normal distribution and use these to initialize the
      model. We only compute p-values up to the p-value of the known maximum
      possible score. Finally, we do not compute the p=0.0 quantile because
      raw anomaly scores are positive and non-zero.
    */
    final double maxScorePvalue = computeLogNormalCdf(maxScore, mu, sigma);
    final double pvalueStep = maxScorePvalue / ((double) numLogNormalQuantiles + 1.0);
    for (double pvalue = pvalueStep; pvalue < maxScorePvalue; pvalue += pvalueStep) {
        double currentScore = computeLogNormalQuantile(pvalue, mu, sigma);
        update(currentScore);
    }
}
 

/**
 * 以close价计算
 *
 * @param close
 * @return
 */
public static double computePriceSTD(double[] close) {
	SummaryStatistics stats = new SummaryStatistics();
	for (int i = 0; i < close.length; i++) {
		stats.addValue(close[i]);
	}
	return stats.getStandardDeviation();
}
 

/**
 * Implements equation (8.13) from "Elementary Statistics: A Problem Solving
 * Approach 4th Edition", Andrew L. Comrey, Howard B. Lee
 *
 * @return the standard error as the ratio of the standard deviation divided
 * by the sqrt(number of users) of the distribution of difference scores.
 */
public double getStandardError() {
    Set<V> overlap = new HashSet<V>(baselineMetricPerDimension.keySet());
    overlap.retainAll(testMetricPerDimension.keySet());

    // paired or matched samples --> analyse distribution of difference scores
    SummaryStatistics differences = new SummaryStatistics();
    for (V key : overlap) {
        double diff = baselineMetricPerDimension.get(key) - testMetricPerDimension.get(key);
        differences.addValue(diff);
    }

    double e = differences.getStandardDeviation() / Math.sqrt(differences.getN());
    return e;
}
 

/**
 * The within-bin smoothing kernel. Returns a Gaussian distribution
 * parameterized by {@code bStats}, unless the bin contains only one
 * observation, in which case a constant distribution is returned.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    if (bStats.getN() == 1) {
        return new ConstantRealDistribution(bStats.getMean());
    } else {
        return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
    }
}
 

/**
 * The within-bin smoothing kernel. Returns a Gaussian distribution
 * parameterized by {@code bStats}, unless the bin contains only one
 * observation, in which case a constant distribution is returned.
 *
 * @param bStats summary statistics for the bin
 * @return within-bin kernel parameterized by bStats
 */
protected RealDistribution getKernel(SummaryStatistics bStats) {
    if (bStats.getN() == 1) {
        return new ConstantRealDistribution(bStats.getMean());
    } else {
        return new NormalDistribution(randomData.getRandomGenerator(),
            bStats.getMean(), bStats.getStandardDeviation(),
            NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
    }
}
 

private static double calcMeanCI(SummaryStatistics stats, double level) {
    try {
        TDistribution tDist = new TDistribution(stats.getN() - 1);
        double critVal = tDist.inverseCumulativeProbability(1.0 - (1 - level) / 2);
        return critVal * stats.getStandardDeviation() / Math.sqrt(stats.getN());
    } catch (MathIllegalArgumentException e) {
        return Double.NaN;
    }
}
 

private BufferedImage toGrayScale(Raster in, PixelCorrection c, 
    boolean invertColors, boolean ignoreBadStats)
 {
    int width = in.getWidth();
    int height = in.getHeight();
    // compute stats
    SummaryStatistics stats = new SummaryStatistics();
    for(int j = 0; j < height; j++)
    {
       for(int i = 0; i < width; i++)
       {
          int pixel = checkAndApplyCorrection(in.getSample(i, j, 0), c);
          if(pixel != c.nodata)
             stats.addValue(pixel);
       }
    }
    double lowerBound = Math.max(
       stats.getMin(), 
       stats.getMean() - 3*stats.getStandardDeviation());
    double upperBound = Math.min(
       stats.getMax(), 
       stats.getMean() + 3*stats.getStandardDeviation());

    if(!ignoreBadStats)
        if(Double.isNaN(stats.getMean()) || 
           Double.isNaN(stats.getStandardDeviation()) || 
           stats.getStandardDeviation() < 1)
            throw new IllegalStateException(
               "Ugly band stats. Acquired during night?");

    return toGrayScale(in, c, invertColors, lowerBound, upperBound);
}
 

/**
 * Applies a global filter on pitches in S1 and moves all pitches whose salience is bellow a certain
 * threshold from S1 to S0. This filter is described in [1], section II-C.
 */
private void applyGlobalFilter() {
    SummaryStatistics statistics = new SummaryStatistics();

    /* Iteration #1: Gather data to obtain salience statistics. */
    for (int t=0; t<this.s1.length; t++) {
        for (int i=0; i<this.s1[t].length; i++) {
            if (this.s1[t][i] == null) {
              continue;
            }
            statistics.addValue(this.s1[t][i].getSalience());
        }
    }

    /* Iteration #2: Move pitches that are bellow the threshold. */
    final double threshold = statistics.getMean() - this.t2 * statistics.getStandardDeviation();
    for (int t=0; t<this.s1.length; t++) {
        for (int i=0; i<this.s1[t].length; i++) {
            if (this.s1[t][i] == null) {
              continue;
            }
            if (this.s1[t][i].getSalience() < threshold) {
                this.moveToS0(t,i);
            }
        }
    }
}
 

public void getSummaryStats(double[] values){
	SummaryStatistics stats = new SummaryStatistics();
	for( int i = 0; i < values.length; i++) {
	        stats.addValue(values[i]);
	}
	double mean = stats.getMean();
	double std = stats.getStandardDeviation();
	System.out.println(mean + "\t" + std);
}
 

@VisibleForTesting
BigDecimal calculateDeviation(Context context, Request request) {

    Instant to = request.getCurrentTime();

    Duration interval = Duration.between(to, request.getTargetTime());

    Instant from = request.getCurrentTime().minus(interval.toMillis() * getSamples(), MILLIS);

    List<Trade> trades = context.listTrades(getKey(context, request), from.minus(interval));

    NavigableMap<Instant, BigDecimal> prices = collapsePrices(trades, interval, from, to, false);

    NavigableMap<Instant, BigDecimal> returns = calculateReturns(prices);

    SummaryStatistics stats = new SummaryStatistics();

    returns.values().stream().filter(Objects::nonNull).forEach(r -> stats.addValue(r.doubleValue()));

    if (stats.getN() <= 1) {
        return null;
    }

    double avg = stats.getMean();

    double dev = stats.getStandardDeviation();

    double sigma = new TDistribution(stats.getN() - 1).inverseCumulativeProbability(PROBABILITY);

    double sum = Math.abs(avg) + (dev * sigma);

    return Double.isFinite(sum) ? BigDecimal.valueOf(sum).setScale(SCALE, HALF_UP) : null;

}