Java Code Examples for org.apache.commons.math3.stat.descriptive.moment.StandardDeviation#evaluate()

The following examples show how to use org.apache.commons.math3.stat.descriptive.moment.StandardDeviation#evaluate() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MathUtilities.java    From StatsAgg with Apache License 2.0 6 votes vote down vote up 
public static BigDecimal computePopulationStandardDeviationOfBigDecimals(List<BigDecimal> numbers) {
    
    if ((numbers == null) || numbers.isEmpty()) {
        return null;
    }
    
    try {
        double[] doublesArray = new double[numbers.size()];

        for (int i = 0; i < doublesArray.length; i++) {
            doublesArray[i] = numbers.get(i).doubleValue();
        }

        StandardDeviation standardDeviation = new StandardDeviation();
        standardDeviation.setBiasCorrected(false);
        BigDecimal standardDeviationResult = new BigDecimal(standardDeviation.evaluate(doublesArray));

        return standardDeviationResult;
    }
    catch (Exception e) {
        logger.error(e.toString() + System.lineSeparator() + StackTrace.getStringFromStackTrace(e));
        return null;
    }
}
Example 2
Source File: UserProfileEigenModeler.java    From Eagle with Apache License 2.0 6 votes vote down vote up 
private void computeStats(RealMatrix m){

        if(m.getColumnDimension() != this.cmdTypes.length){
            LOG.error("Please fix the commands list in config file");
            return;
        }
        statistics = new UserCommandStatistics[m.getColumnDimension()];
        for(int i=0; i<m.getColumnDimension(); i++){
            UserCommandStatistics stats = new UserCommandStatistics();
            stats.setCommandName(this.cmdTypes[i]);
            RealVector colData = m.getColumnVector(i);
            StandardDeviation deviation = new StandardDeviation();
            double stddev = deviation.evaluate(colData.toArray());
            //LOG.info("stddev is nan?" + (stddev == Double.NaN? "yes":"no"));
            if(stddev <= lowVarianceVal)
                stats.setLowVariant(true);
            else
                stats.setLowVariant(false);
            stats.setStddev(stddev);
            Mean mean = new Mean();
            double mu = mean.evaluate(colData.toArray());
            //LOG.info("mu is nan?" + (mu == Double.NaN? "yes":"no"));
            stats.setMean(mu);
            statistics[i] = stats;
        }
    }
Example 3
Source File: TestLongStdDevPopAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up 
@Override
protected Number getExpectedValue(int start, int length)
{
    if (length == 0) {
        return null;
    }

    double[] values = new double[length];
    for (int i = 0; i < length; i++) {
        values[i] = start + i;
    }

    StandardDeviation stdDev = new StandardDeviation(false);
    return stdDev.evaluate(values);
}
Example 4
Source File: TestDoubleStdDevPopAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up 
@Override
protected Number getExpectedValue(int start, int length)
{
    if (length == 0) {
        return null;
    }

    double[] values = new double[length];
    for (int i = 0; i < length; i++) {
        values[i] = start + i;
    }

    StandardDeviation stdDev = new StandardDeviation(false);
    return stdDev.evaluate(values);
}
Example 5
Source File: TestDoubleStdDevAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up 
@Override
protected Number getExpectedValue(int start, int length)
{
    if (length < 2) {
        return null;
    }

    double[] values = new double[length];
    for (int i = 0; i < length; i++) {
        values[i] = start + i;
    }

    StandardDeviation stdDev = new StandardDeviation();
    return stdDev.evaluate(values);
}
Example 6
Source File: TestLongStdDevAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up 
@Override
protected Number getExpectedValue(int start, int length)
{
    if (length < 2) {
        return null;
    }

    double[] values = new double[length];
    for (int i = 0; i < length; i++) {
        values[i] = start + i;
    }

    StandardDeviation stdDev = new StandardDeviation();
    return stdDev.evaluate(values);
}
Example 7
Source File: ExplicitIDWithLRISRouterIT.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up 
private void assertDataIsDistributedAccordingWithDBIDRouting(Node node)
{
    int [] shardIdentifiers = range(0, 15).toArray();
    int shardCount = shardIdentifiers.length;
    int howManyDocumentsPerShard = 10000;

    Map<Integer, Integer> nodeDistributionMap = new HashMap<>();

    range(0, shardCount * howManyDocumentsPerShard)
            .mapToLong(Long::valueOf)
            .forEach(id -> {
                node.setId(id);
                stream(shardIdentifiers)
                        .forEach(shardId -> {
                            if (router.routeNode(shardCount, shardId, node))
                            {
                                nodeDistributionMap.merge(shardId, 1, Integer::sum);
                            }
                        });
            });

    StandardDeviation sd = new StandardDeviation();
    double deviation = sd.evaluate(nodeDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());
    double norm = deviation/(howManyDocumentsPerShard) * 100;

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());

    // Asserts the standard deviation of the distribution map is in percentage lesser than 30%
    assertTrue(
            nodeDistributionMap.values().toString() + ", SD = " + deviation + ", SD_NORM = " + norm + "%",
            norm < 30);
}
Example 8
Source File: PropertyRouterIT.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up 
@Test
public void onlyPatternIsNull_shouldBalanceNodesOnShardProperty()
{
    int [] shardIdentifiers = range(0,15).toArray();
    int shardCount = shardIdentifiers.length;
    int howManyDocumentsPerShard = 10000;

    Map<Integer, Integer> nodeDistributionMap = new HashMap<>();

    range(0, shardCount * howManyDocumentsPerShard)
            .mapToLong(Long::valueOf)
            .forEach(id -> {
                Node node = new Node();
                node.setShardPropertyValue(String.valueOf(id));
                stream(shardIdentifiers)
                        .forEach(shardId -> {
                            if (router.routeNode(shardCount, shardId, node))
                            {
                                nodeDistributionMap.merge(shardId, 1, Integer::sum);
                            }
                        });
            });

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());
    StandardDeviation sd = new StandardDeviation();
    double deviation = sd.evaluate(nodeDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());
    double norm = (deviation/howManyDocumentsPerShard) * 100;

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());

    // Asserts the standard deviation of the distribution map is in percentage lesser than 30%
    assertTrue(
            nodeDistributionMap.values().toString() + ", SD = " + deviation + ", SD_NORM = " + norm + "%",
            norm < 30);

}
Example 9
Source File: PropertyRouterIT.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up 
@Test
public void propertyAndPatternArentNull_shouldBalanceNodesOnShardProperty()
{
    int [] shardIdentifiers = range(0,15).toArray();
    int shardCount = shardIdentifiers.length;
    int howManyDocumentsPerShard = 100000;

    Map<Integer, Integer> nodeDistributionMap = new HashMap<>();

    router = new PropertyRouter("([0-9]*)(_)");
    router.fallback = fallback;

    range(0, shardCount * howManyDocumentsPerShard)
            .mapToLong(Long::valueOf)
            .forEach(id -> {
                Node node = new Node();
                node.setShardPropertyValue(id + "_ignoreThisPart");
                stream(shardIdentifiers)
                        .forEach(shardId -> {
                            if (router.routeNode(shardCount, shardId, node))
                            {
                                nodeDistributionMap.merge(shardId, 1, Integer::sum);
                            }
                        });
            });

    StandardDeviation sd = new StandardDeviation();
    double deviation = sd.evaluate(nodeDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());
    double norm = deviation/(howManyDocumentsPerShard) * 100;

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());

    // Asserts the standard deviation of the distribution map is in percentage lesser than 30%
    assertTrue(
            nodeDistributionMap.values().toString() + ", SD = " + deviation + ", SD_NORM = " + norm + "%",
            norm < 30);
}
Example 10
Source File: DBIDRouterTest.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up 
@Test
public void multipleShardsInTheCluster_shouldBalanceNodes()
{
    int [] shardIdentifiers = range(0,15).toArray();
    int shardCount = shardIdentifiers.length;
    int howManyDocumentsPerShard = 10000;

    Map<Integer, Integer> nodeDistributionMap = new HashMap<>();

    range(0, shardCount * howManyDocumentsPerShard)
            .mapToLong(Long::valueOf)
            .forEach(id -> {
                Node node = new Node();
                node.setId(id);
                stream(shardIdentifiers)
                        .forEach(shardId -> {
                            if (router.routeNode(shardCount, shardId, node))
                            {
                                nodeDistributionMap.merge(shardId, 1, Integer::sum);
                            }
                        });
            });

    StandardDeviation sd = new StandardDeviation();
    double deviation = sd.evaluate(nodeDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());
    double norm = deviation/(howManyDocumentsPerShard) * 100;

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());

    // Asserts the standard deviation of the distribution map is in percentage lesser than 30%
    assertTrue(
            nodeDistributionMap.values().toString() + ", SD = " + deviation + ", SD_NORM = " + norm + "%",
            norm < 30);
}
Example 11
Source File: UserProfileKDEModeler.java    From Eagle with Apache License 2.0 5 votes vote down vote up 
private void computeStats(RealMatrix m){
    if(m.getColumnDimension() !=  this.cmdTypes.length){
        LOG.error("Please fix the commands list in config file");
    }

    statistics = new UserCommandStatistics[m.getColumnDimension()];

    for(int i=0; i<m.getColumnDimension(); i++){
        UserCommandStatistics stats = new UserCommandStatistics();
        stats.setCommandName(this.cmdTypes[i]);
        RealVector colData = m.getColumnVector(i);
        StandardDeviation deviation = new StandardDeviation();
        double stddev = deviation.evaluate(colData.toArray());

        if(LOG.isDebugEnabled()) LOG.debug("Stddev is NAN ? " + (Double.isNaN(stddev) ? "yes" : "no"));
        if(stddev <= lowVarianceVal)
            stats.setLowVariant(true);
        else
            stats.setLowVariant(false);

        stats.setStddev(stddev);
        Mean mean = new Mean();
        double mu = mean.evaluate(colData.toArray());
        if(LOG.isDebugEnabled()) LOG.debug("mu is NAN ? " + (Double.isNaN(mu)? "yes":"no"));

        stats.setMean(mu);
        statistics[i]=stats;
    }
}
Example 12
Source File: ACLIDMurmurRouterTest.java    From SearchServices with GNU Lesser General Public License v3.0 4 votes vote down vote up 
@Test
public void sevenShardsInTheCluster_shouldBalanceNodesAndAcls()
{
    int [] shardIdentifiers = range(0,7).toArray();
    int shardCount = shardIdentifiers.length;
    int howManyDocumentsPerShard = 1000;

    // Maps used for validating the data distribution
    Map<Integer, Integer> aclDistributionMap = new HashMap<>();
    Map<Integer, Integer> nodeDistributionMap = new HashMap<>();

    range(0, shardCount * howManyDocumentsPerShard)
            .mapToLong(Long::valueOf)
            .forEach(id -> {
                Acl acl = new Acl(randomPositiveInteger(), id);
                Node node = new Node();
                node.setAclId(acl.getId());

                stream(shardIdentifiers)
                        .forEach(shardId -> {
                            if (router.routeAcl(shardCount, shardId, acl))
                            {
                                aclDistributionMap.merge(shardId, 1, Integer::sum);
                            }

                            if (router.routeNode(shardCount, shardId, node))
                            {
                                nodeDistributionMap.merge(shardId, 1, Integer::sum);
                            }
                        });
            });

    StandardDeviation sd = new StandardDeviation();
    double aclsDeviation = sd.evaluate(aclDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());
    double nodesDeviation = sd.evaluate(nodeDistributionMap.values().stream().mapToDouble(Number::doubleValue).toArray());

    assertEquals(shardIdentifiers.length, nodeDistributionMap.size());
    assertEquals(shardIdentifiers.length, aclDistributionMap.size());

    // Asserts the standard deviation of the distribution map is in percentage lesser than 30%
    assertTrue(aclDistributionMap.values().toString() + ", SD = " + aclsDeviation, aclsDeviation/(howManyDocumentsPerShard) * 100 < 30);
    assertTrue(nodeDistributionMap.values().toString() + ", SD = " + nodesDeviation,nodesDeviation/(howManyDocumentsPerShard) * 100 < 30);
}