Java Code Examples for org.apache.commons.math3.stat.correlation.PearsonsCorrelation#getCorrelationMatrix()
The following examples show how to use
org.apache.commons.math3.stat.correlation.PearsonsCorrelation#getCorrelationMatrix() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CorrelationAnalysisEngine.java From TomboloDigitalConnector with MIT License | 6 votes |
|
/**
* Calculates pearson correlation between pair of columns in the in the matrix, assuming that there is a strict
* one to one relationship between the matrix columns and the field specifications in the list.
*
* Writes the correlation, pValue and standard error to a file using JSON format.
*
* @param matrix the input matrix where fields are represented by as columns and subjects by rows
* @param fields a list of field specifications for which the correlations are to be calculated
* @param correlationAnalysisOutputPath is the file to which the results are written
* @throws Exception
*/
public static void calculateAndOutputCorrelations(RealMatrix matrix, List<FieldRecipe> fields,
String correlationAnalysisOutputPath) throws Exception {
PearsonsCorrelation correlation = new PearsonsCorrelation(matrix);
RealMatrix correlationMatrix = correlation.getCorrelationMatrix();
RealMatrix pValueMatrix = correlation.getCorrelationPValues();
RealMatrix standardErrorMatrix = correlation.getCorrelationStandardErrors();
// Output the correlation analysis
JSONArray correlationArray = new JSONArray();
for (int i=0; i<correlationMatrix.getRowDimension(); i++){
for (int j=0; j<correlationMatrix.getColumnDimension(); j++){
JSONObject correlationObject = new JSONObject();
correlationObject.put("xFieldLabel", fields.get(i).toField().getLabel());
correlationObject.put("yFieldLabel", fields.get(j).toField().getLabel());
correlationObject.put("correlationCoefficient", correlationMatrix.getEntry(i,j));
correlationObject.put("pValue", pValueMatrix.getEntry(i,j));
correlationObject.put("standardError", standardErrorMatrix.getEntry(i,j));
correlationArray.add(correlationObject);
}
}
Writer writer = new OutputStreamWriter(new FileOutputStream(correlationAnalysisOutputPath), "UTF-8");
writer.write(correlationArray.toJSONString());
writer.flush();
writer.close();
}
Example 2
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = FastMath.log(airdata[3][i]);
x[i][2] = FastMath.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = FastMath.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 3
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 4
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 5
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 6
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 7
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 8
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = Math.log(airdata[3][i]);
x[i][2] = Math.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = Math.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (Math.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (Math.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (Math.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
Example 9
| Source File: MillerUpdatingRegressionTest.java From astor with GNU General Public License v2.0 | 4 votes |
|
@Test
public void testPCorr() {
MillerUpdatingRegression instance = new MillerUpdatingRegression(4, false);
double[][] x = new double[airdata[0].length][];
double[] y = new double[airdata[0].length];
double[] cp = new double[10];
double[] yxcorr = new double[4];
double[] diag = new double[4];
double sumysq = 0.0;
int off = 0;
for (int i = 0; i < airdata[0].length; i++) {
x[i] = new double[4];
x[i][0] = 1.0;
x[i][1] = FastMath.log(airdata[3][i]);
x[i][2] = FastMath.log(airdata[4][i]);
x[i][3] = airdata[5][i];
y[i] = FastMath.log(airdata[2][i]);
off = 0;
for (int j = 0; j < 4; j++) {
double tmp = x[i][j];
for (int k = 0; k <= j; k++, off++) {
cp[off] += tmp * x[i][k];
}
yxcorr[j] += tmp * y[i];
}
sumysq += y[i] * y[i];
}
PearsonsCorrelation pearson = new PearsonsCorrelation(x);
RealMatrix corr = pearson.getCorrelationMatrix();
off = 0;
for (int i = 0; i < 4; i++, off += (i + 1)) {
diag[i] = FastMath.sqrt(cp[off]);
}
instance.addObservations(x, y);
double[] pc = instance.getPartialCorrelations(0);
int idx = 0;
off = 0;
int off2 = 6;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < i; j++) {
if (FastMath.abs(pc[idx] - cp[off] / (diag[i] * diag[j])) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
++off;
}
++off;
if (FastMath.abs(pc[i+off2] - yxcorr[ i] / (FastMath.sqrt(sumysq) * diag[i])) > 1.0e-8) {
Assert.fail("Assert.failed cross product i = " + i + " y");
}
}
double[] pc2 = instance.getPartialCorrelations(1);
idx = 0;
for (int i = 1; i < 4; i++) {
for (int j = 1; j < i; j++) {
if (FastMath.abs(pc2[idx] - corr.getEntry(j, i)) > 1.0e-8) {
Assert.fail("Failed cross products... i = " + i + " j = " + j);
}
++idx;
}
}
double[] pc3 = instance.getPartialCorrelations(2);
if (pc3 == null) {
Assert.fail("Should not be null");
}
return;
}
