org.apache.commons.math3.stat.correlation.PearsonsCorrelation Java Examples

The following examples show how to use org.apache.commons.math3.stat.correlation.PearsonsCorrelation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CorrelationAnalysisEngine.java    From TomboloDigitalConnector with MIT License 6 votes vote down vote up
/**
 * Calculates pearson correlation between pair of columns in the in the matrix, assuming that there is a strict
 * one to one relationship between the matrix columns and the field specifications in the list.
 *
 * Writes the correlation, pValue and standard error to a file using JSON format.
 *
 * @param matrix the input matrix where fields are represented by as columns and subjects by rows
 * @param fields a list of field specifications for which the correlations are to be calculated
 * @param correlationAnalysisOutputPath is the file to which the results are written
 * @throws Exception
 */
public static void calculateAndOutputCorrelations(RealMatrix matrix, List<FieldRecipe> fields,
                                                   String correlationAnalysisOutputPath) throws Exception {
    PearsonsCorrelation correlation = new PearsonsCorrelation(matrix);
    RealMatrix correlationMatrix = correlation.getCorrelationMatrix();
    RealMatrix pValueMatrix = correlation.getCorrelationPValues();
    RealMatrix standardErrorMatrix = correlation.getCorrelationStandardErrors();

    // Output the correlation analysis
    JSONArray correlationArray = new JSONArray();
    for (int i=0; i<correlationMatrix.getRowDimension(); i++){
        for (int j=0; j<correlationMatrix.getColumnDimension(); j++){
            JSONObject correlationObject = new JSONObject();
            correlationObject.put("xFieldLabel", fields.get(i).toField().getLabel());
            correlationObject.put("yFieldLabel", fields.get(j).toField().getLabel());
            correlationObject.put("correlationCoefficient", correlationMatrix.getEntry(i,j));
            correlationObject.put("pValue", pValueMatrix.getEntry(i,j));
            correlationObject.put("standardError", standardErrorMatrix.getEntry(i,j));
            correlationArray.add(correlationObject);
        }
    }
    Writer writer = new OutputStreamWriter(new FileOutputStream(correlationAnalysisOutputPath), "UTF-8");
    writer.write(correlationArray.toJSONString());
    writer.flush();
    writer.close();
}
 
Example #2
Source File: CorrelationSignificanceEvaluator.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Object doWork(Object value) throws IOException{
  if(null == value){
    return null;
  } else if(value instanceof Matrix) {
    Matrix matrix = (Matrix) value;
    Object corr = matrix.getAttribute("corr");
    if(corr instanceof PearsonsCorrelation) {
      PearsonsCorrelation pcorr = (PearsonsCorrelation)corr;
      RealMatrix  realMatrix = pcorr.getCorrelationPValues();
      return new Matrix(realMatrix.getData());
    } else {
      throw new IOException("Correlation pvalues are only available for Pearsons and Spearmans correlations");
    }
  } else {
    throw new IOException("matrix parameter expected for transpose function");
  }
}
 
Example #3
Source File: TestDoubleCorrelationAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
protected Object getExpectedValue(int start, int length)
{
    if (length <= 1) {
        return null;
    }
    PearsonsCorrelation corr = new PearsonsCorrelation();
    return corr.correlation(constructDoublePrimitiveArray(start + 2, length), constructDoublePrimitiveArray(start, length));
}
 
Example #4
Source File: CorrelationCalculator.java    From ADW with GNU General Public License v3.0 5 votes vote down vote up
public static double getPearson(List<Double> list1, List<Double> list2)
{
	PearsonsCorrelation correlation = new PearsonsCorrelation();
	double c = correlation.correlation(getArray(list1),getArray(list2));
		
	return c;
}
 
Example #5
Source File: CalibrationLinearityActivity.java    From NoiseCapture with GNU General Public License v3.0 5 votes vote down vote up
/**
 * @return Pearson's product-moment correlation coefficients for the measured data
 */
private double[] computePearson() {
    if(freqLeqStats.size() < 3) {
        return null;
    }
    // Frequency count, one dataset by frequency
    int dataSetCount = freqLeqStats.get(freqLeqStats.size() - 1).whiteNoiseLevel.getdBaLevels().length;
    double[] pearsonCoefficient = new double[dataSetCount];

    StringBuilder log = new StringBuilder();
    for(int freqId = 0; freqId < dataSetCount; freqId++) {
        double[] xValues = new double[freqLeqStats.size()];
        double[] yValues = new double[freqLeqStats.size()];
        int idStep = 0;
        for(LinearCalibrationResult result : freqLeqStats) {
            double dbLevel = result.measure[freqId].getLeqMean();
            double whiteNoise = result.whiteNoiseLevel.getdBaLevels()[freqId];
            xValues[idStep] = whiteNoise;
            yValues[idStep] = dbLevel;
            if(freqId == 0) {
                LOGGER.info("100 hZ white noise " + whiteNoise + " dB spl: " + dbLevel+ " dB");
            }
            idStep++;
        }
        pearsonCoefficient[freqId] = new PearsonsCorrelation().correlation(xValues, yValues);
        if(log.length() == 0) {
            log.append("[");
        } else {
            log.append(", ");
        }
        log.append(String.format(Locale.getDefault(), "%.2f %%",pearsonCoefficient[freqId] * 100 ));
    }
    log.append("]");
    LOGGER.info("Pearson's values : "+log.toString());
    return pearsonCoefficient;
}
 
Example #6
Source File: NumberColumnTest.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrelation2() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, NaN, 9, 10};
  double[] y = new double[] {1, 2, 3, NaN, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 
Example #7
Source File: NumberColumnTest.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrelation() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
  double[] y = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultS = xCol.spearmans(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new SpearmansCorrelation().correlation(x, y), resultS, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 
Example #8
Source File: NumericColumn.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
default double autoCorrelation(int lag) {
  int slice = this.size() - lag;
  if (slice <= 1) {
    return Double.NaN;
  }
  NumericColumn<?> x = (NumericColumn<?>) this.first(slice);
  NumericColumn<?> y = (NumericColumn<?>) this.last(slice);
  return new PearsonsCorrelation().correlation(x.asDoubleArray(), y.asDoubleArray());
}
 
Example #9
Source File: NumberColumnTest.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrelation2() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, NaN, 9, 10};
  double[] y = new double[] {1, 2, 3, NaN, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 
Example #10
Source File: NumberColumnTest.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrelation() {
  double[] x = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
  double[] y = new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};

  DoubleColumn xCol = DoubleColumn.create("x", x);
  DoubleColumn yCol = DoubleColumn.create("y", y);

  double resultP = xCol.pearsons(yCol);
  double resultS = xCol.spearmans(yCol);
  double resultK = xCol.kendalls(yCol);
  assertEquals(new PearsonsCorrelation().correlation(x, y), resultP, 0.0001);
  assertEquals(new SpearmansCorrelation().correlation(x, y), resultS, 0.0001);
  assertEquals(new KendallsCorrelation().correlation(x, y), resultK, 0.0001);
}
 
Example #11
Source File: NumericColumn.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
default double autoCorrelation(int lag) {
  int slice = this.size() - lag;
  if (slice <= 1) {
    return Double.NaN;
  }
  NumericColumn<?> x = (NumericColumn<?>) this.first(slice);
  NumericColumn<?> y = (NumericColumn<?>) this.last(slice);
  return new PearsonsCorrelation().correlation(x.asDoubleArray(), y.asDoubleArray());
}
 
Example #12
Source File: CorrelationEvaluatorTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void test() throws IOException {
  double[] l1 = new double[] {3.4, 4.5, 6.7};
  double[] l2 = new double[] {1.2, 3.2, 3};
  
  values.clear();
  values.put("l1", l1);
  values.put("l2", l2);
  
  Assert.assertEquals(new PearsonsCorrelation().correlation(l1, l2), factory.constructEvaluator("corr(l1,l2)").evaluate(new Tuple(values)));
}
 
Example #13
Source File: TestRealCorrelationAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
protected Object getExpectedValue(int start, int length)
{
    if (length <= 1) {
        return null;
    }
    PearsonsCorrelation corr = new PearsonsCorrelation();
    return (float) corr.correlation(constructDoublePrimitiveArray(start + 2, length), constructDoublePrimitiveArray(start, length));
}
 
Example #14
Source File: TestDoubleCorrelationAggregation.java    From presto with Apache License 2.0 5 votes vote down vote up
private void testNonTrivialAggregation(double[] y, double[] x)
{
    PearsonsCorrelation corr = new PearsonsCorrelation();
    double expected = corr.correlation(x, y);
    checkArgument(Double.isFinite(expected) && expected != 0.0 && expected != 1.0, "Expected result is trivial");
    testAggregation(expected, createDoublesBlock(box(y)), createDoublesBlock(box(x)));
}
 
Example #15
Source File: StatsUtil.java    From MeteoInfo with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
 * Calculates a Pearson correlation coefficient.
 *
 * @param x X data
 * @param y Y data
 * @return Pearson correlation and p-value.
 */
public static double[] pearsonr(Array x, Array y) {
    x = x.copyIfView();
    y = y.copyIfView();

    if (ArrayMath.containsNaN(x) || ArrayMath.containsNaN(y)) {
        Array[] xy = ArrayMath.removeNaN(x, y);
        if (xy == null) {
            return new double[]{Double.NaN, Double.NaN};
        }
        
        x = xy[0];
        y = xy[1];
    }
    
    if (MAMath.isEqual(x, y)) {
        return new double[]{1, 0};
    }
    
    int m = (int)x.getSize();
    int n = 1;
    double[][] aa = new double[m][n * 2];
    for (int i = 0; i < m; i++) {
        for (int j = 0; j < n * 2; j++) {
            if (j < n) {
                aa[i][j] = x.getDouble(i * n + j);
            } else {
                aa[i][j] = y.getDouble(i * n + j - n);
            }
        }
    }
    RealMatrix matrix = new Array2DRowRealMatrix(aa, false);
    PearsonsCorrelation pc = new PearsonsCorrelation(matrix);
    double r = pc.getCorrelationMatrix().getEntry(0, 1);
    double pvalue = pc.getCorrelationPValues().getEntry(0, 1);
    return new double[]{r, pvalue};
}
 
Example #16
Source File: LR.java    From ml-models with Apache License 2.0 5 votes vote down vote up
@UserFunction(value = "regression.linear.correlation")
@Description("Calculate Pearson's correlation coefficient between first and second data lists")
public double correlation(@Name("first") List<Double> first, @Name("second") List<Double> second) {
    double[] firstArray = doubleListToArray(first);
    double[] secondArray = doubleListToArray(second);
    return new PearsonsCorrelation().correlation(firstArray, secondArray);
}
 
Example #17
Source File: NumericColumn.java    From tablesaw with Apache License 2.0 4 votes vote down vote up
/** Returns the pearson's correlation between the receiver and the otherColumn */
default double pearsons(NumericColumn<?> otherColumn) {
  double[] x = asDoubleArray();
  double[] y = otherColumn.asDoubleArray();
  return new PearsonsCorrelation().correlation(x, y);
}
 
Example #18
Source File: Similarity.java    From mzmine3 with GNU General Public License v2.0 4 votes vote down vote up
@Override
public double calc(double[][] data) {
  PearsonsCorrelation corr = new PearsonsCorrelation();
  return corr.correlation(col(data, 0), col(data, 1));
}
 
Example #19
Source File: Similarity.java    From mzmine2 with GNU General Public License v2.0 4 votes vote down vote up
@Override
public double calc(double[][] data) {
  PearsonsCorrelation corr = new PearsonsCorrelation();
  return corr.correlation(col(data, 0), col(data, 1));
}
 
Example #20
Source File: PearsonTest.java    From Java-Data-Science-Cookbook with MIT License 4 votes vote down vote up
public void calculatePearson(double[] x, double[] y){
	PearsonsCorrelation pCorrelation = new PearsonsCorrelation();
	double cor = pCorrelation.correlation(x, y);//take out false too
	System.out.println(cor);
}
 
Example #21
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = FastMath.log(airdata[3][i]);
        x[i][2] = FastMath.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = FastMath.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #22
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #23
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #24
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #25
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #26
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #27
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = Math.log(airdata[3][i]);
        x[i][2] = Math.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = Math.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #28
Source File: MillerUpdatingRegressionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test
public void testPCorr() {
    MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
    double[][] x = new double[airdata[0].length][];
    double[] y = new double[airdata[0].length];
    double[] cp = new double[10];
    double[] yxcorr = new double[4];
    double[] diag = new double[4];
    double sumysq = 0.0;
    int off = 0;
    for (int i = 0; i < airdata[0].length; i++) {
        x[i] = new double[4];
        x[i][0] = 1.0;
        x[i][1] = FastMath.log(airdata[3][i]);
        x[i][2] = FastMath.log(airdata[4][i]);
        x[i][3] = airdata[5][i];
        y[i] = FastMath.log(airdata[2][i]);
        off = 0;
        for (int j = 0; j < 4; j++) {
            double tmp = x[i][j];
            for (int k = 0; k <= j; k++, off++) {
                cp[off] += tmp * x[i][k];
            }
            yxcorr[j] += tmp * y[i];
        }
        sumysq += y[i] * y[i];
    }
    PearsonsCorrelation pearson = new PearsonsCorrelation(x);
    RealMatrix corr = pearson.getCorrelationMatrix();
    off = 0;
    for (int i = 0; i < 4; i++, off += (i + 1)) {
        diag[i] = FastMath.sqrt(cp[off]);
    }

    instance.addObservations(x, y);
    double[] pc = instance.getPartialCorrelations(0);
    int idx = 0;
    off = 0;
    int off2 = 6;
    for (int i = 0; i < 4; i++) {
        for (int j = 0; j < i; j++) {
            if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
            ++off;
        }
        ++off;
        if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
            Assert.fail("Assert.failed cross product i = " + i + " y");
        }
    }
    double[] pc2 = instance.getPartialCorrelations(1);

    idx = 0;

    for (int i = 1; i < 4; i++) {
        for (int j = 1; j < i; j++) {
            if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
                Assert.fail("Failed cross products... i = " + i + " j = " + j);
            }
            ++idx;
        }
    }
    double[] pc3 = instance.getPartialCorrelations(2);
    if (pc3 == null) {
        Assert.fail("Should not be null");
    }
    return;
}
 
Example #29
Source File: NumericColumn.java    From tablesaw with Apache License 2.0 4 votes vote down vote up
/** Returns the pearson's correlation between the receiver and the otherColumn */
default double pearsons(NumericColumn<?> otherColumn) {
  double[] x = asDoubleArray();
  double[] y = otherColumn.asDoubleArray();
  return new PearsonsCorrelation().correlation(x, y);
}