org.apache.commons.math3.stat.StatUtils Java Examples

@Override
public Object doWork(Object value){
  if(null == value){
    return null;
  }
  else if(value instanceof List){
    return Arrays.stream(StatUtils.normalize(((List<?>)value).stream().mapToDouble(innerValue -> ((Number)innerValue).doubleValue()).toArray())).boxed().collect(Collectors.toList());
  } else if (value instanceof Matrix) {
    Matrix matrix = (Matrix) value;
    double[][] data = matrix.getData();
    double[][] standardized = new double[data.length][];
    for(int i=0; i<data.length; i++) {
      double[] row = data[i];
      standardized[i] = StatUtils.normalize(row);
    }
    Matrix m = new Matrix(standardized);
    m.setRowLabels(matrix.getRowLabels());
    m.setColumnLabels(matrix.getColumnLabels());
    return m;
  } else {
    return doWork(Arrays.asList((BigDecimal)value));
  }
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Test
public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    Assert.assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    Assert.assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Override
public Object doWork(Object value) throws IOException{
  if(null == value){
    throw new IOException(String.format(Locale.ROOT, "Unable to find %s(...) because the value is null", constructingFactory.getFunctionName(getClass())));
  }
  else if(value instanceof List){
    @SuppressWarnings({"unchecked"})
    List<Number> c = (List<Number>) value;
    double[] data = new double[c.size()];
    for(int i=0; i< c.size(); i++) {
      data[i] = c.get(i).doubleValue();
    }

    double[] mode = StatUtils.mode(data);
    List<Number> l = new ArrayList<>();
    for(double d : mode) {
      l.add(d);
    }

    return l;
  }
  else{
    throw new IOException(String.format(Locale.ROOT, "Unable to find %s(...) because the value is not a collection, instead a %s was found", constructingFactory.getFunctionName(getClass()), value.getClass().getSimpleName()));
  }
}
 

@Test
public void testSample() {
    final double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    final int length = values.length;
    final double mean = StatUtils.mean(values); // 6.333...
    final SemiVariance sv = new SemiVariance();  // Default bias correction is true
    final double downsideSemiVariance = sv.evaluate(values); // Downside is the default
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {-2d, 2d, 4d, -2d, 3d, 5d}, mean) / (length - 1),
            downsideSemiVariance, 1E-14);

    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upsideSemiVariance = sv.evaluate(values);
    Assert.assertEquals(TestUtils.sumSquareDev(new double[] {22d, 11d, 14d}, mean) / (length - 1),
            upsideSemiVariance, 1E-14);

    // Verify that upper + lower semivariance against the mean sum to variance
    Assert.assertEquals(StatUtils.variance(values), downsideSemiVariance + upsideSemiVariance, 10e-12);
}
 

@Test
public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    Assert.assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    Assert.assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

public double calculateOverallAnnotationSufficiencyForAttributeSet(Set<OWLClass> atts) throws UnknownOWLClassException {
	SummaryStatistics stats = computeAttributeSetSimilarityStats(atts);
	if ((this.getSummaryStatistics() == null) || Double.isNaN(this.getSummaryStatistics().mean.getMean())) {
		LOG.info("Stats have not been computed yet - doing this now");
		this.computeSystemStats();
	}
	// score = mean(atts)/mean(overall) + max(atts)/max(overall) + sum(atts)/mean(sum(overall))
	double overall_score = 0.0;
	Double mean_score = stats.getMean();
	Double max_score = stats.getMax();
	Double sum_score = stats.getSum();
	if (!(mean_score.isNaN() || max_score.isNaN() || sum_score.isNaN())) {
		mean_score = StatUtils.min(new double[]{(mean_score / this.overallSummaryStatsPerIndividual.mean.getMean()),1.0});
		max_score = StatUtils.min(new double[]{(max_score / this.overallSummaryStatsPerIndividual.max.getMax()),1.0});
		sum_score = StatUtils.min(new double[]{(sum_score / this.overallSummaryStatsPerIndividual.sum.getMean()),1.0});
		overall_score = (mean_score + max_score + sum_score) / 3;		
	}
	LOG.info("Overall mean: "+mean_score + " max: "+max_score + " sum:"+sum_score + " combined:"+overall_score);
	return overall_score;
}
 

@Override
public Object doWork(Object value) throws IOException{
    if(null == value){
        throw new IOException(String.format(Locale.ROOT, "Unable to find %s(...) because the value is null", constructingFactory.getFunctionName(getClass())));
    }
    else if(value instanceof List){
        @SuppressWarnings({"unchecked"})
        List<Number> c = (List<Number>) value;
        double[] data = new double[c.size()];
        for(int i=0; i< c.size(); i++) {
            data[i] = c.get(i).doubleValue();
        }

        return StatUtils.variance(data);
    }
    else{
        throw new IOException(String.format(Locale.ROOT, "Unable to find %s(...) because the value is not a collection, instead a %s was found", constructingFactory.getFunctionName(getClass()), value.getClass().getSimpleName()));
    }
}
 

@Test
public void testMinMax() {
    da.addElement(2.0);
    da.addElement(22.0);
    da.addElement(-2.0);
    da.addElement(21.0);
    da.addElement(22.0);
    da.addElement(42.0);
    da.addElement(62.0);
    da.addElement(22.0);
    da.addElement(122.0);
    da.addElement(1212.0);

    Assert.assertEquals("Min should be -2.0", -2.0, StatUtils.min(da.getElements()), Double.MIN_VALUE);
    Assert.assertEquals(
        "Max should be 1212.0",
        1212.0,
        StatUtils.max(da.getElements()),
        Double.MIN_VALUE);
}
 

/**
 * Given a set of likelihoods in log10, output 1- the probability of the most largest likelihood [ 1-p].
 * @param allLikelihoods a collection of likelihoods
 * @return 1 - (best like / sum (likes))
 */
public double getOneMinusPvalueFromLog10Likelihood (final double [] allLikelihoods) {
	// we clone the array so we don't change it.
	double [] likes=allLikelihoods.clone();
	Arrays.sort(likes);

	double maxValue = StatUtils.max(likes);

	double totalLikelihood=0;
	double allButBestLikelihood=0;

	for (int i=0; i<likes.length; i++) {
		double d = likes[i];
		d=d-maxValue;
		d=Math.pow(10, d);
		totalLikelihood+=d;
		if (i!=(likes.length-1))
			allButBestLikelihood+=d;
	}

	double result = allButBestLikelihood/totalLikelihood;
	if (result < Double.MIN_VALUE) result = Double.MIN_VALUE;
	return result;
}
 

@Test
void testMomentsSampling() {
    final UniformRandomProvider rng = RandomSource.create(RandomSource.WELL_1024_A,
                                                          123456789L);
    final int numSamples = 1000;
    for (final double alpha : ALPHA_BETAS) {
        for (final double beta : ALPHA_BETAS) {
            final BetaDistribution betaDistribution = new BetaDistribution(alpha, beta);
            final double[] observed = AbstractContinuousDistribution.sample(numSamples,
                    betaDistribution.createSampler(rng));
            Arrays.sort(observed);

            Assertions.assertEquals(betaDistribution.getMean(), StatUtils.mean(observed),
                                    EPSILON, () -> String.format("E[Beta(%.2f, %.2f)]", alpha, beta));
            Assertions.assertEquals(betaDistribution.getVariance(), StatUtils.variance(observed),
                                    EPSILON, () -> String.format("Var[Beta(%.2f, %.2f)]", alpha, beta));
        }
    }
}
 

/**
 * If alpha = 2, than it must be Gaussian distribution
 */
@Test
public void testGaussianCase() {
    StableRandomGenerator generator = new StableRandomGenerator(rg, 2d, 0.0);

    double[] sample = new double[sampleSize];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.02);
    Assert.assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

/**
 * If alpha = 2, than it must be Gaussian distribution
 */
@Test
public void testGaussianCase() {
    StableRandomGenerator generator = new StableRandomGenerator(rg, 2d, 0.0);

    double[] sample = new double[sampleSize];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.02);
    Assert.assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

/**
 * Run the double nextDouble() method test Due to leptokurtic property the
 * acceptance range is widened.
 * 
 * TODO: verify that tolerance this wide is really OK
 */
@Test
public void testNextDouble() {
    StableRandomGenerator generator = new StableRandomGenerator(rg, 1.3,
            0.1);
    double[] sample = new double[2 * sampleSize];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.3);
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
@Test
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    Assert.assertEquals(variance, lower + upper, 10e-12);
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
@Test
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    Assert.assertEquals(variance, lower + upper, 10e-12);
}
 

/**
 * Check that the lower + upper semivariance against the mean sum to the
 * variance.
 */
@Test
public void testVarianceDecompMeanCutoff() {
    double[] values = { -2.0d, 2.0d, 4.0d, -2.0d, 22.0d, 11.0d, 3.0d, 14.0d, 5.0d };
    double variance = StatUtils.variance(values);
    SemiVariance sv = new SemiVariance(true); // Bias corrected
    sv.setVarianceDirection(SemiVariance.DOWNSIDE_VARIANCE);
    final double lower = sv.evaluate(values);
    sv.setVarianceDirection(SemiVariance.UPSIDE_VARIANCE);
    final double upper = sv.evaluate(values);
    Assert.assertEquals(variance, lower + upper, 10e-12);
}
 

/**
 * Verifies that calculateYVariance and calculateResidualVariance return consistent
 * values with direct variance computation from Y, residuals, respectively.
 */
protected void checkVarianceConsistency(OLSMultipleLinearRegression model) {
    // Check Y variance consistency
    TestUtils.assertEquals(StatUtils.variance(model.getY().toArray()), model.calculateYVariance(), 0);
    
    // Check residual variance consistency
    double[] residuals = model.calculateResiduals().toArray();
    RealMatrix X = model.getX();
    TestUtils.assertEquals(
            StatUtils.variance(model.calculateResiduals().toArray()) * (residuals.length - 1),
            model.calculateErrorVariance() * (X.getRowDimension() - X.getColumnDimension()), 1E-20);
    
}
 

private void summarizeSkeleton(final SkeletonResult sr) {
	final String TABLE_TITLE = "Summary of Rendered Paths";
	final ResultsTable rt = getTable(TABLE_TITLE);
	try {
		double sumLength = 0d;
		final int[] branches = sr.getBranches();
		final double[] avgLengths = sr.getAverageBranchLength();
		for (int i = 0; i < sr.getNumOfTrees(); i++)
			sumLength += avgLengths[i] * branches[i];
		rt.incrementCounter();
		rt.addValue("N. Rendered Paths", renderingPaths.size());
		rt.addValue("Unit", imp.getCalibration().getUnits());
		rt.addValue("Total length", sumLength);
		rt.addValue("Mean branch length", StatUtils.mean(avgLengths));
		rt.addValue("Length of longest branch", StatUtils.max(sr.getMaximumBranchLength()));
		rt.addValue("# Branches", IntStream.of(sr.getBranches()).sum());
		rt.addValue("# Junctions", IntStream.of(sr.getJunctions()).sum());
		rt.addValue("# End-points", IntStream.of(sr.getEndPoints()).sum());
		rt.addValue("Fitering", getFilterString());
		if (restrictByRoi && roi != null && roi.isArea())
			rt.addValue("ROI Name", roi.getName() == null ? "Unammed ROI" : roi.getName());

	} catch (final Exception ignored) {
		SNT.error("Some statistics could not be calculated.");
	} finally {
		rt.show(TABLE_TITLE);
	}
}
 

@Test
public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    GaussianRandomGenerator generator = new GaussianRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.012);
    Assert.assertEquals(1.0, StatUtils.variance(sample), 0.01);
}
 

/**
 * Run the double nextDouble() method test Due to leptokurtic property the
 * acceptance range is widened.
 * 
 * TODO: verify that tolerance this wide is really OK
 */
@Test
public void testNextDouble() {
    StableRandomGenerator generator = new StableRandomGenerator(rg, 1.3,
            0.1);
    double[] sample = new double[2 * sampleSize];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.3);
}
 

@Test
void testPercentileFunctions() {
  double[] values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
  DoubleColumn c = DoubleColumn.create("test", values);
  c.appendCell("");

  assertEquals(1, countMissing.summarize(c), 0.0001);
  assertEquals(11, countWithMissing.summarize(c), 0.0001);

  assertEquals(StatUtils.percentile(values, 90), percentile90.summarize(c), 0.0001);
  assertEquals(StatUtils.percentile(values, 95), percentile95.summarize(c), 0.0001);
  assertEquals(StatUtils.percentile(values, 99), percentile99.summarize(c), 0.0001);

  assertEquals(10, countUnique.summarize(c), 0.0001);
}
 

@Test
public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.07);
    Assert.assertEquals(1.0, StatUtils.variance(sample), 0.02);
}
 

@Override
public float getResult() {
    if (count == 0)
        return Float.NaN;
    
    double[] primitiveValues = new double[floatValues.size()];
    for (int i = 0; i < floatValues.size(); i++)
        primitiveValues[i] = floatValues.get(i);

    return (float) StatUtils.mode(primitiveValues)[0];
}
 

/**
 * 
 * @param dataActiveMap
 * @param sample
 */
public void calculateStats ( boolean[] dataActiveMap, double[] sample ) {
    ArrayList<Double> liveSample = new ArrayList<>();
    sampleMean = 0.0;

    if ( sample.length > 0 ) {

        for (int i = 0; i < sample.length; i ++) {
            if ( dataActiveMap[i] ) {
                sampleMean += sample[i];
                liveSample.add( sample[i] );
            }
        }

        sampleMean /= liveSample.size();
    }
    
    double[] liveSampleArray = new double[liveSample.size()];
    for (int i = 0; i < liveSampleArray.length; i ++){
        liveSampleArray[i] = (double)liveSample.get( i );
    }
    
           
    sampleMean = StatUtils.mean( liveSampleArray );
    variance = StatUtils.variance( liveSampleArray );

    stdErrSampleMean = Math.sqrt(variance) / Math.sqrt( liveSampleArray.length );

}
 

/**
 * Returns a double array, whose values are quantiles from the given source, based on the given
 * size. The idea is to produce size elements that represent the quantiles of source array
 *
 * @param source The array to whose quantiles are calculated
 * @param size The size of the array to return
 */
private static double[] interpolate(double[] source, int size) {
  double[] interpolatedData = new double[size];
  for (int i = 0; i < size; i++) {
    double value = ((i + .5) / (double) size) * 100;
    interpolatedData[i] = StatUtils.percentile(source, value);
  }
  return interpolatedData;
}
 

@Test
public void testMeanAndStandardDeviation() {
    RandomGenerator rg = new JDKRandomGenerator();
    rg.setSeed(17399225432l);
    UniformRandomGenerator generator = new UniformRandomGenerator(rg);
    double[] sample = new double[10000];
    for (int i = 0; i < sample.length; ++i) {
        sample[i] = generator.nextNormalizedDouble();
    }
    Assert.assertEquals(0.0, StatUtils.mean(sample), 0.07);
    Assert.assertEquals(1.0, StatUtils.variance(sample), 0.02);
}