Java Code Examples for org.apache.commons.math3.stat.descriptive.SummaryStatistics#addValue()

/**
 * This method calls the method that actually does the calculations (except
 * P-value).
 *
 * @param categoryData
 *            <code>Collection</code> of <code>double[]</code> arrays each
 *            containing data for one category
 * @return computed AnovaStats
 * @throws NullArgumentException
 *             if <code>categoryData</code> is <code>null</code>
 * @throws DimensionMismatchException
 *             if the length of the <code>categoryData</code> array is less
 *             than 2 or a contained <code>double[]</code> array does not
 *             contain at least two values
 */
private AnovaStats anovaStats(final Collection<double[]> categoryData)
    throws NullArgumentException, DimensionMismatchException {

    MathUtils.checkNotNull(categoryData);

    final Collection<SummaryStatistics> categoryDataSummaryStatistics =
            new ArrayList<SummaryStatistics>(categoryData.size());

    // convert arrays to SummaryStatistics
    for (final double[] data : categoryData) {
        final SummaryStatistics dataSummaryStatistics = new SummaryStatistics();
        categoryDataSummaryStatistics.add(dataSummaryStatistics);
        for (final double val : data) {
            dataSummaryStatistics.addValue(val);
        }
    }

    return anovaStats(categoryDataSummaryStatistics, false);

}
 

private void tstDoubleGen(double tolerance)throws Exception {
    empiricalDistribution2.load(dataArray);
    empiricalDistribution2.reSeed(1000);
    SummaryStatistics stats = new SummaryStatistics();
    for (int i = 1; i < 1000; i++) {
        stats.addValue(empiricalDistribution2.getNextValue());
    }
    Assert.assertEquals("mean", 5.069831575018909, stats.getMean(), tolerance);
    Assert.assertEquals("std dev", 1.0173699343977738, stats.getStandardDeviation(), tolerance);
}
 

@Test
public void testDoubleDirect() {
    SummaryStatistics sample = new SummaryStatistics();
    final int N = 10000;
    for (int i = 0; i < N; ++i) {
        sample.addValue(generator.nextDouble());
    }
    Assert.assertEquals("Note: This test will fail randomly about 1 in 100 times.",
            0.5, sample.getMean(), FastMath.sqrt(N/12.0) * 2.576);
    Assert.assertEquals(1.0 / (2.0 * FastMath.sqrt(3.0)),
                 sample.getStandardDeviation(), 0.01);
}
 

@Override
public void processSegment(SegmentContainer shot) {
  if (shot.getMostRepresentativeFrame() == VideoFrame.EMPTY_VIDEO_FRAME) {
    return;
  }
  if (!phandler.idExists(shot.getId())) {
    SummaryStatistics[] stats = new SummaryStatistics[256];
    for (int i = 0; i < 256; ++i) {
      stats[i] = new SummaryStatistics();
    }
    List<VideoFrame> videoFrames = shot.getVideoFrames();
    List<Boolean> edgePixels = new ArrayList<>();
    for (VideoFrame f : videoFrames) {
      MultiImage img = f.getImage();
      edgePixels = EdgeImg.getEdgePixels(img, edgePixels);
      ArrayList<LinkedList<Boolean>> partition = GridPartitioner.partition(edgePixels,
          img.getWidth(), img.getHeight(), 16, 16);
      for (int i = 0; i < partition.size(); ++i) {
        LinkedList<Boolean> edge = partition.get(i);
        SummaryStatistics stat = stats[i];
        for (boolean b : edge) {
          stat.addValue(b ? 1 : 0);
        }
      }
    }
    float[] result = new float[64];
    for (int i = 0; i < 64; ++i) {
      result[i] = (float) stats[i].getMean();
    }
    persist(shot.getId(), new FloatVectorImpl(result));
  }
}
 

private static double calculateSimilarity(String[] nameWords, String[] tokens, boolean full) {
  if (!full || nameWords.length >= tokens.length) {
    int matches = countMatches(tokens, nameWords);
    SummaryStatistics statistics = new SummaryStatistics();
    statistics.addValue(matches / (double) nameWords.length);
    statistics.addValue(matches / (double) tokens.length);
    return statistics.getMean();
  } else {
    return 0.0;
  }
}
 

@Test
    public void testAnalyticalDifferentiation()
        throws MaxCountExceededException, DimensionMismatchException,
               NumberIsTooSmallException, NoBracketingException,
               UnknownParameterException, MismatchedEquations {
        AbstractIntegrator integ =
            new DormandPrince54Integrator(1.0e-8, 100.0, new double[] { 1.0e-4, 1.0e-4 }, new double[] { 1.0e-4, 1.0e-4 });
        SummaryStatistics residualsP0 = new SummaryStatistics();
        SummaryStatistics residualsP1 = new SummaryStatistics();
        for (double b = 2.88; b < 3.08; b += 0.001) {
            Brusselator brusselator = new Brusselator(b);
            double[] z = { 1.3, b };
            double[][] dZdZ0 = new double[2][2];
            double[]   dZdP  = new double[2];

            JacobianMatrices jacob = new JacobianMatrices(brusselator, Brusselator.B);
            jacob.addParameterJacobianProvider(brusselator);
            jacob.setInitialParameterJacobian(Brusselator.B, new double[] { 0.0, 1.0 });

            ExpandableStatefulODE efode = new ExpandableStatefulODE(brusselator);
            efode.setTime(0);
            efode.setPrimaryState(z);
            jacob.registerVariationalEquations(efode);

            integ.setMaxEvaluations(5000);
            integ.integrate(efode, 20.0);
            jacob.getCurrentMainSetJacobian(dZdZ0);
            jacob.getCurrentParameterJacobian(Brusselator.B, dZdP);
//            Assert.assertEquals(5000, integ.getMaxEvaluations());
//            Assert.assertTrue(integ.getEvaluations() > 350);
//            Assert.assertTrue(integ.getEvaluations() < 510);
            residualsP0.addValue(dZdP[0] - brusselator.dYdP0());
            residualsP1.addValue(dZdP[1] - brusselator.dYdP1());
        }
        Assert.assertTrue((residualsP0.getMax() - residualsP0.getMin()) < 0.014);
        Assert.assertTrue(residualsP0.getStandardDeviation() < 0.003);
        Assert.assertTrue((residualsP1.getMax() - residualsP1.getMin()) < 0.05);
        Assert.assertTrue(residualsP1.getStandardDeviation() < 0.01);
    }
 

/** {@inheritDoc} */
@Override
public void computeStats() throws IOException {
    String str = null;
    double val = 0.0;
    sampleStats = new SummaryStatistics();
    while ((str = inputStream.readLine()) != null) {
        val = Double.valueOf(str).doubleValue();
        sampleStats.addValue(val);
    }
    inputStream.close();
    inputStream = null;
}
 

public static double computePriceChangeSTD(double[] open, double[] close) {
	SummaryStatistics stats = new SummaryStatistics();
	for (int i = 0; i < open.length; i++) {
		stats.addValue(close[i] - open[i]);
	}
	return stats.getStandardDeviation();
}
 

/** {@inheritDoc} */
@Override
public void computeStats() throws IOException {
    String str = null;
    double val = 0.0;
    sampleStats = new SummaryStatistics();
    while ((str = inputStream.readLine()) != null) {
        val = Double.parseDouble(str);
        sampleStats.addValue(val);
    }
    inputStream.close();
    inputStream = null;
}
 

@Test
public void testDoubleDirect() {
    SummaryStatistics sample = new SummaryStatistics();
    final int N = 10000;
    for (int i = 0; i < N; ++i) {
        sample.addValue(generator.nextDouble());
    }
    Assert.assertEquals("Note: This test will fail randomly about 1 in 100 times.",
            0.5, sample.getMean(), FastMath.sqrt(N/12.0) * 2.576);
    Assert.assertEquals(1.0 / (2.0 * FastMath.sqrt(3.0)),
                 sample.getStandardDeviation(), 0.01);
}
 

/** {@inheritDoc} */
@Override
public void computeBinStats() throws IOException {
    for (int i = 0; i < inputArray.length; i++) {
        SummaryStatistics stats =
            binStats.get(findBin(inputArray[i]));
        stats.addValue(inputArray[i]);
    }
}
 

private void computeAverageRepairabilityForVioGenQueries(Map<VioGenQuery, List<VioGenQueryCellChange>> changesForQueryMap) {
    for (VioGenQuery vioGenQuery : changesForQueryMap.keySet()) {
        RepairabilityStats repairabilityStats = new RepairabilityStats();
        SummaryStatistics stats = new SummaryStatistics();
        List<VioGenQueryCellChange> changesForQuery = changesForQueryMap.get(vioGenQuery);
        for (VioGenQueryCellChange change : changesForQuery) {
            stats.addValue(change.getRepairability());
        }
        repairabilityStats.setMean(stats.getMean());
        double confidenceInterval = calcMeanCI(stats, 0.95);
        repairabilityStats.setConfidenceInterval(confidenceInterval);
        if (logger.isInfoEnabled()) logger.info("Repairability for query " + vioGenQuery.toShortString() + ": " + repairabilityStats);
        ErrorGeneratorStats.getInstance().getVioGenQueriesRepairability().put(vioGenQuery, repairabilityStats);
    }
}
 

private void computeAverageRepairabilityForVioDependencies(Map<Dependency, List<VioGenQueryCellChange>> changes) {
    for (Dependency dependency : changes.keySet()) {
        RepairabilityStats repairabilityStats = new RepairabilityStats();
        SummaryStatistics stats = new SummaryStatistics();
        List<VioGenQueryCellChange> changesForQuery = changes.get(dependency);
        for (VioGenQueryCellChange change : changesForQuery) {
            stats.addValue(change.getRepairability());
        }
        repairabilityStats.setMean(stats.getMean());
        double confidenceInterval = calcMeanCI(stats, 0.95);
        repairabilityStats.setConfidenceInterval(confidenceInterval);
        if (logger.isInfoEnabled()) logger.info("Repairability for query " + dependency.getId() + ": " + repairabilityStats);
        ErrorGeneratorStats.getInstance().getDependencyRepairability().put(dependency, repairabilityStats);
    }
}
 

@Test
    public void testAnalyticalDifferentiation() {
        AbstractIntegrator integ =
            new DormandPrince54Integrator(1.0e-8, 100.0, new double[] { 1.0e-4, 1.0e-4 }, new double[] { 1.0e-4, 1.0e-4 });
        SummaryStatistics residualsP0 = new SummaryStatistics();
        SummaryStatistics residualsP1 = new SummaryStatistics();
        for (double b = 2.88; b < 3.08; b += 0.001) {
            Brusselator brusselator = new Brusselator(b);
            double[] z = { 1.3, b };
            double[][] dZdZ0 = new double[2][2];
            double[]   dZdP  = new double[2];

            JacobianMatrices jacob = new JacobianMatrices(brusselator, Brusselator.B);
            jacob.addParameterJacobianProvider(brusselator);
            jacob.setInitialParameterJacobian(Brusselator.B, new double[] { 0.0, 1.0 });

            ExpandableStatefulODE efode = new ExpandableStatefulODE(brusselator);
            efode.setTime(0);
            efode.setPrimaryState(z);
            jacob.registerVariationalEquations(efode);

            integ.setMaxEvaluations(5000);
            integ.integrate(efode, 20.0);
            jacob.getCurrentMainSetJacobian(dZdZ0);
            jacob.getCurrentParameterJacobian(Brusselator.B, dZdP);
//            Assert.assertEquals(5000, integ.getMaxEvaluations());
//            Assert.assertTrue(integ.getEvaluations() > 350);
//            Assert.assertTrue(integ.getEvaluations() < 510);
            residualsP0.addValue(dZdP[0] - brusselator.dYdP0());
            residualsP1.addValue(dZdP[1] - brusselator.dYdP1());
        }
        Assert.assertTrue((residualsP0.getMax() - residualsP0.getMin()) < 0.014);
        Assert.assertTrue(residualsP0.getStandardDeviation() < 0.003);
        Assert.assertTrue((residualsP1.getMax() - residualsP1.getMin()) < 0.05);
        Assert.assertTrue(residualsP1.getStandardDeviation() < 0.01);
    }
 

@Before
public void setUp() {
    tooShortStats = new SummaryStatistics();
    tooShortStats.addValue(0d);
}
 

@Before
public void setUp() {
    tooShortStats = new SummaryStatistics();
    tooShortStats.addValue(0d);
}
 

@Test
    public void testInternalDifferentiation() {
        AbstractIntegrator integ =
            new DormandPrince54Integrator(1.0e-8, 100.0, new double[] { 1.0e-4, 1.0e-4 }, new double[] { 1.0e-4, 1.0e-4 });
        double hP = 1.0e-12;
        double hY = 1.0e-12;
        SummaryStatistics residualsP0 = new SummaryStatistics();
        SummaryStatistics residualsP1 = new SummaryStatistics();
        for (double b = 2.88; b < 3.08; b += 0.001) {
            ParamBrusselator brusselator = new ParamBrusselator(b);
            brusselator.setParameter(ParamBrusselator.B, b);
            double[] z = { 1.3, b };
            double[][] dZdZ0 = new double[2][2];
            double[]   dZdP  = new double[2];

            JacobianMatrices jacob = new JacobianMatrices(brusselator, new double[] { hY, hY }, ParamBrusselator.B);
            jacob.setParameterizedODE(brusselator);
            jacob.setParameterStep(ParamBrusselator.B, hP);
            jacob.setInitialParameterJacobian(ParamBrusselator.B, new double[] { 0.0, 1.0 });

            ExpandableStatefulODE efode = new ExpandableStatefulODE(brusselator);
            efode.setTime(0);
            efode.setPrimaryState(z);
            jacob.registerVariationalEquations(efode);

            integ.setMaxEvaluations(5000);
            integ.integrate(efode, 20.0);
            jacob.getCurrentMainSetJacobian(dZdZ0);
            jacob.getCurrentParameterJacobian(ParamBrusselator.B, dZdP);
//            Assert.assertEquals(5000, integ.getMaxEvaluations());
//            Assert.assertTrue(integ.getEvaluations() > 1500);
//            Assert.assertTrue(integ.getEvaluations() < 2100);
//            Assert.assertEquals(4 * integ.getEvaluations(), integ.getEvaluations());
            residualsP0.addValue(dZdP[0] - brusselator.dYdP0());
            residualsP1.addValue(dZdP[1] - brusselator.dYdP1());
        }
        Assert.assertTrue((residualsP0.getMax() - residualsP0.getMin()) < 0.02);
        Assert.assertTrue(residualsP0.getStandardDeviation() < 0.003);
        Assert.assertTrue((residualsP1.getMax() - residualsP1.getMin()) < 0.05);
        Assert.assertTrue(residualsP1.getStandardDeviation() < 0.01);
    }
 

public void benchmarkNodeLost() throws Exception {
  List<String> results = new ArrayList<>();
  for (int wait : renard5x) {
    for (int delay : renard5x) {
      SummaryStatistics totalTime = new SummaryStatistics();
      SummaryStatistics ignoredOurEvents = new SummaryStatistics();
      SummaryStatistics ignoredOtherEvents = new SummaryStatistics();
      SummaryStatistics startedOurEvents = new SummaryStatistics();
      SummaryStatistics startedOtherEvents = new SummaryStatistics();
      for (int i = 0; i < 5; i++) {
        if (cluster != null) {
          cluster.close();
        }
        setUp();
        setupTest();
        long total = doTestNodeLost(wait, delay * 1000, 0);
        totalTime.addValue(total);
        // get event counts
        Map<String, Map<String, AtomicInteger>> counts = cluster.simGetEventCounts();
        Map<String, AtomicInteger> map = counts.remove("node_lost_trigger");
        startedOurEvents.addValue(map.getOrDefault("STARTED", ZERO).get());
        ignoredOurEvents.addValue(map.getOrDefault("IGNORED", ZERO).get());
        int otherStarted = 0;
        int otherIgnored = 0;
        for (Map<String, AtomicInteger> m : counts.values()) {
          otherStarted += m.getOrDefault("STARTED", ZERO).get();
          otherIgnored += m.getOrDefault("IGNORED", ZERO).get();
        }
        startedOtherEvents.addValue(otherStarted);
        ignoredOtherEvents.addValue(otherIgnored);
      }
      results.add(String.format(Locale.ROOT, "%d\t%d\t%4.0f\t%4.0f\t%4.0f\t%4.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f",
          wait, delay, startedOurEvents.getMean(), ignoredOurEvents.getMean(),
          startedOtherEvents.getMean(), ignoredOtherEvents.getMean(),
          totalTime.getMin(), totalTime.getMax(), totalTime.getMean(), totalTime.getStandardDeviation(), totalTime.getVariance()));
    }
  }
  log.info("===== RESULTS ======");
  log.info("waitFor\tdelay\tSTRT\tIGN\toSTRT\toIGN\tmin\tmax\tmean\tstdev\tvar");
  if (log.isInfoEnabled()) {
    results.forEach(s -> log.info(s));
  }
}
 

@Before
public void setUp() {
    tooShortStats = new SummaryStatistics();
    tooShortStats.addValue(0d);
}