Python numpy.corrcoef() Examples

def get_corr_func(method):
    if method in ['kendall', 'spearman']:
        from scipy.stats import kendalltau, spearmanr
    elif callable(method):
        return method

    def _pearson(a, b):
        return np.corrcoef(a, b)[0, 1]

    def _kendall(a, b):
        rs = kendalltau(a, b)
        if isinstance(rs, tuple):
            return rs[0]
        return rs

    def _spearman(a, b):
        return spearmanr(a, b)[0]

    _cor_methods = {
        'pearson': _pearson,
        'kendall': _kendall,
        'spearman': _spearman
    }
    return _cor_methods[method] 

def _calc_score(self, X):

        if isinstance(self.criterion, str):

            if self.criterion == "maxmin":
                D = cdist(X, X)
                np.fill_diagonal(D, np.inf)
                return np.min(D)

            elif self.criterion == "correlation":
                M = np.corrcoef(X.T, rowvar=True)
                return -np.sum(np.tril(M, -1) ** 2)

            else:
                raise Exception("Unknown criterion.")
        elif callable(self.criterion):
            return self.criterion(X)

        else:
            raise Exception("Either provide a str or a function as a criterion!") 

def test_2d_with_missing(self):
        # Test corrcoef on 2D variable w/ missing value
        x = self.data
        x[-1] = masked
        x = x.reshape(3, 4)

        test = corrcoef(x)
        control = np.corrcoef(x)
        assert_almost_equal(test[:-1, :-1], control[:-1, :-1])
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, "bias and ddof have no effect")
            # ddof and bias have no or negligible effect on the function
            assert_almost_equal(corrcoef(x, ddof=-2)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, ddof=3)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, bias=1)[:-1, :-1],
                                control[:-1, :-1]) 

def test_2d_with_missing(self):
        # Test corrcoef on 2D variable w/ missing value
        x = self.data
        x[-1] = masked
        x = x.reshape(3, 4)

        test = corrcoef(x)
        control = np.corrcoef(x)
        assert_almost_equal(test[:-1, :-1], control[:-1, :-1])
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, "bias and ddof have no effect")
            # ddof and bias have no or negligible effect on the function
            assert_almost_equal(corrcoef(x, ddof=-2)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, ddof=3)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, bias=1)[:-1, :-1],
                                control[:-1, :-1]) 

def X_corrLoadings(self):
        """
        Returns array holding correlation loadings of array X. First column
        holds correlation loadings for component 1, second column holds
        correlation loadings for component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_XcorrLoadings = np.zeros((np.shape(self.arrT)[1], np.shape(self.arrP)[0]), float)

        # Compute correlation loadings:
        # For each PC in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.arrX)[1]):
                origVar = self.arrX[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_XcorrLoadings[PC, var] = corrs[0,1]

        self.arr_XcorrLoadings = np.transpose(arr_XcorrLoadings)

        return self.arr_XcorrLoadings 

def Y_corrLoadings(self):
        """
        Returns array holding correlation loadings of array X. First column
        holds correlation loadings for component 1, second column holds
        correlation loadings for component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_YcorrLoadings = np.zeros((np.shape(self.arrT)[1], np.shape(self.arrQ)[0]), float)

        # Compute correlation loadings:
        # For each PC in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.arrY)[1]):
                origVar = self.arrY[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_YcorrLoadings[PC, var] = corrs[0,1]

        self.arr_YcorrLoadings = np.transpose(arr_YcorrLoadings)

        return self.arr_YcorrLoadings 

def X_corrLoadings(self):
        """
        Returns array holding correlation loadings of array X. First column
        holds correlation loadings for component 1, second column holds
        correlation loadings for component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_corrLoadings = np.zeros((np.shape(self.arrT)[1], np.shape(self.arrP)[0]), float)

        # Compute correlation loadings:
        # For each component in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.arrX)[1]):
                origVar = self.arrX[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_corrLoadings[PC, var] = corrs[0,1]

        self.arr_corrLoadings = np.transpose(arr_corrLoadings)

        return self.arr_corrLoadings 

def Y_corrLoadings(self):
        """
        Returns array holding correlation loadings of array X. First column
        holds correlation loadings for component 1, second column holds
        correlation loadings for component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_YcorrLoadings = np.zeros((np.shape(self.arrT)[1], np.shape(self.arrQ)[0]), float)

        # Compute correlation loadings:
        # For each component in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.arrY)[1]):
                origVar = self.arrY[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_YcorrLoadings[PC, var] = corrs[0,1]

        self.arr_YcorrLoadings = np.transpose(arr_YcorrLoadings)

        return self.arr_YcorrLoadings 

def X_corrLoadings(self):
        """
        Returns array holding correlation loadings of array X. First column
        holds correlation loadings for component 1, second column holds
        correlation loadings for component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_corrLoadings = np.zeros((np.shape(self.arrT)[1],
                                     np.shape(self.arrP)[0]), float)

        # Compute correlation loadings:
        # For each component in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.arrX)[1]):
                origVar = self.arrX[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_corrLoadings[PC, var] = corrs[0, 1]

        self.arr_corrLoadings = np.transpose(arr_corrLoadings)

        return self.arr_corrLoadings 

def Y_corrLoadings(self):
        """
        Returns an array holding correlation loadings of vector y. Columns
        represent components. First column for component 1, second columns for
        component 2, etc.
        """

        # Creates empty matrix for correlation loadings
        arr_ycorrLoadings = np.zeros((np.shape(self.arrT)[1], np.shape(self.arrQ)[0]), float)

        # Compute correlation loadings:
        # For each PC in score matrix
        for PC in range(np.shape(self.arrT)[1]):
            PCscores = self.arrT[:, PC]

            # For each variable/attribute in original matrix (not meancentered)
            for var in range(np.shape(self.vecy)[1]):
                origVar = self.vecy[:, var]
                corrs = np.corrcoef(PCscores, origVar)
                arr_ycorrLoadings[PC, var] = corrs[0,1]

        self.arr_ycorrLoadings = np.transpose(arr_ycorrLoadings)

        return self.arr_ycorrLoadings 

def correlation(obj1, obj2, start=0, end=-1, price_feature='Close'):
    if isinstance(obj1, str) or isinstance(obj2, str):
        obj1 = log_price_returns(obj1, start, end, price_feature)
        obj2 = log_price_returns(obj2, start, end, price_feature)
        # simple and rough treatment: assume biz days are the same among the two tickers
        if len(obj1)>len(obj2):
            obj1 = obj1[len(obj1)-len(obj2):]
        else:
            obj2 = obj2[len(obj2)-len(obj1):]
        start = 0
        end = -1

    if end < 0:
        end += len(obj1)
    if start < 0:
        start += len(obj1)

    return np.corrcoef(obj1[start: (end + 1)], obj2[start: (end + 1)])[0, 1] 

def test_2d_w_missing(self):
        # Test corrcoef on 2D variable w/ missing value
        x = self.data
        x[-1] = masked
        x = x.reshape(3, 4)

        test = corrcoef(x)
        control = np.corrcoef(x)
        assert_almost_equal(test[:-1, :-1], control[:-1, :-1])
        with catch_warn_mae():
            warnings.simplefilter("ignore")
            # ddof and bias have no or negligible effect on the function
            assert_almost_equal(corrcoef(x, ddof=-2)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, ddof=3)[:-1, :-1],
                                control[:-1, :-1])
            assert_almost_equal(corrcoef(x, bias=1)[:-1, :-1],
                                control[:-1, :-1]) 

def _lhscorrelate(n, samples, iterations):
    mincorr = np.inf

    # Minimize the components correlation coefficients
    for i in range(iterations):
        # Generate a random LHS
        Hcandidate = _lhsclassic(n, samples)
        R = np.corrcoef(Hcandidate)
        if np.max(np.abs(R[R != 1])) < mincorr:
            mincorr = np.max(np.abs(R - np.eye(R.shape[0])))
            print(
            'new candidate solution found with max,abs corrcoef = {}'.format(
                mincorr))
            H = Hcandidate.copy()

    return H


################################################################################ 

def test_atlas_connectivity(betaseries_file, atlas_file, atlas_lut):
    # read in test files
    bs_data = nib.load(str(betaseries_file)).get_data()
    atlas_lut_df = pd.read_csv(str(atlas_lut), sep='\t')

    # expected output
    pcorr = np.corrcoef(bs_data.squeeze())
    np.fill_diagonal(pcorr, np.NaN)
    regions = atlas_lut_df['regions'].values
    pcorr_df = pd.DataFrame(pcorr, index=regions, columns=regions)
    expected_zcorr_df = pcorr_df.apply(lambda x: (np.log(1 + x) - np.log(1 - x)) * 0.5)

    # run instance of AtlasConnectivity
    ac = AtlasConnectivity(timeseries_file=str(betaseries_file),
                           atlas_file=str(atlas_file),
                           atlas_lut=str(atlas_lut))

    res = ac.run()

    output_zcorr_df = pd.read_csv(res.outputs.correlation_matrix,
                                  na_values='n/a',
                                  delimiter='\t',
                                  index_col=0)

    os.remove(res.outputs.correlation_matrix)
    # test equality of the matrices up to 3 decimals
    pd.testing.assert_frame_equal(output_zcorr_df, expected_zcorr_df,
                                  check_less_precise=3) 

def test_mesh(self):
        '''
        test_mesh() ensures that many general mesh properties and methods are working.
        '''
        import neuropythy.geometry as geo
        logging.info('neuropythy: Testing meshes and properties...')
        # get a random subject's mesh
        sub  = ny.data['benson_winawer_2018'].subjects['S1204']
        hem  = sub.hemis[('lh','rh')[np.random.randint(2)]]
        msh  = hem.white_surface
        # few simple things
        self.assertEqual(msh.coordinates.shape[0], 3)
        self.assertEqual(msh.tess.faces.shape[0], 3)
        self.assertEqual(msh.tess.edges.shape[0], 2)
        self.assertEqual(msh.vertex_count, msh.coordinates.shape[1])
        # face areas and edge lengths should all be non-negative
        self.assertGreaterEqual(np.min(msh.face_areas), 0)
        self.assertGreaterEqual(np.min(msh.edge_lengths), 0)
        # test the properties
        self.assertTrue('blerg' in msh.with_prop(blerg=msh.prop('curvature')).properties)
        self.assertFalse('curvature' in msh.wout_prop('curvature').properties)
        self.assertEqual(msh.properties.row_count, msh.vertex_count)
        self.assertLessEqual(np.abs(np.mean(msh.prop('curvature'))), 0.1)
        # use the property interface to grab a fancy masked property
        v123_areas = msh.property('midgray_surface_area',
                                  mask=('inf-prf_visual_area', (1,2,3)),
                                  null=0)
        v123_area = np.sum(v123_areas)
        self.assertLessEqual(v123_area, 15000)
        self.assertGreaterEqual(v123_area, 500)
        (v1_ecc, v1_rad) = msh.property(['prf_eccentricity','prf_radius'],
                                        mask=('inf-prf_visual_area', 1),
                                        weights='prf_variance_explained',
                                        weight_min=0.1,
                                        clipped=0,
                                        null=np.nan)
        wh = np.isfinite(v1_ecc) & np.isfinite(v1_rad)
        self.assertGreater(np.corrcoef(v1_ecc[wh], v1_rad[wh])[0,0], 0.5) 

def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`
            The labels of the data.
        preds : list of `NDArray`
            Predicted values.
        """
        labels, preds = check_label_shapes(labels, preds, True)

        for label, pred in zip(labels, preds):
            check_label_shapes(label, pred, False, True)
            label = label.asnumpy()
            pred = pred.asnumpy()
            self.sum_metric += numpy.corrcoef(pred.ravel(), label.ravel())[0, 1]
            self.num_inst += 1 

def test_pearsonr():
    pred = mx.nd.array([[0.7, 0.3], [0.1, 0.9], [1., 0]])
    label = mx.nd.array([[0, 1], [1, 0], [1, 0]])
    pearsonr_expected = np.corrcoef(pred.asnumpy().ravel(), label.asnumpy().ravel())[0, 1]
    metric = mx.metric.create('pearsonr')
    metric.update([label], [pred])
    _, pearsonr = metric.get()
    assert pearsonr == pearsonr_expected 

def ncc(ypred, y):
    return np.corrcoef(ypred, y)[1,0] 

def ncc(ypred, y):
    return np.corrcoef(ypred, y)[1,0] 

def _pears_sim(inA, inB):
    if len(inA) < 3:
        return 1.0
    return 0.5 + 0.5 * np.corrcoef(inA, inB, rowvar=0)[0][1]


# 余弦相关范围-1->+1 越大越相似 

def make_seed_corr(self, summary_df, network, func_fnifti, temp_dir):
        self.seed_corr = os.path.join(temp_dir, 'scorr{}{}.dscalar.nii'.format(
                self.vert_type, network))
        meants = self.dataframe.loc[:, summary_df.loc[:, 'NETWORK'] ==
                network].mean(axis=1)

        temp_nifti_seed = os.path.join(temp_dir, 'seedcorr{}.nii.gz'.format(
                network))
        ## correlated the mean timeseries with the func data
        out = np.zeros([func_fnifti.dims[0]*func_fnifti.dims[1]*func_fnifti.dims[2],
                1])
        ## determine brainmask bits..
        std_array = np.std(func_fnifti.data, axis=1)
        std_nonzero = np.where(std_array > 0)[0]
        mask_indices = std_nonzero
        for i in mask_indices:
            out[i] = np.corrcoef(meants, func_fnifti.data[i, :])[0][1]
        ## reshape data and write it out to a fake nifti file
        out = out.reshape([func_fnifti.dims[0], func_fnifti.dims[1],
                func_fnifti.dims[2], 1])
        out = nib.nifti1.Nifti1Image(out, func_fnifti.affine)
        out.to_filename(temp_nifti_seed)

        ## convert back
        run(['wb_command','-cifti-convert','-from-nifti',
                temp_nifti_seed, func_fnifti.template, self.seed_corr, '-reset-scalars'])

        run(['wb_command', '-cifti-palette', self.seed_corr,
                'MODE_AUTO_SCALE_PERCENTAGE', self.seed_corr,
                '-palette-name', 'PSYCH-NO-NONE'])
        if not os.path.exists(self.seed_corr):
            logger.error("Could not generate seed corr file {} for {}"
                    "".format(self.seed_corr, self.vert_type))
            sys.exit(1) 

def mass_partial_corr(X,massY,Z):
    ''' mass partial correlation between X and many Y signals after regressing Z from both sides
    Parameters
    -----------
    X : 1D predictor vector (n observations)
    massY : 2D numpy matrix of signals to correlate (k signals by n observations)
    Z : 2D numpy matrix of signals to regress from both X and Y (n observations by p confounds)
    Returns
    -----------
    1D vector or partial correlations (k signals long)
    '''

    assert X.shape[0]==massY.shape[1]
    assert massY.shape[1]==Z.shape[0]

    ## stack X and Y together to prepare to regress
    pre_res = np.vstack((X,massY))

    ## loop over all signals and to regress out confounds
    res_by_z = np.zeros(pre_res.shape) -1
    for i in range(pre_res.shape[0]):
        res_by_z[i,:] = linalg_calc_residulals(Z, pre_res[i,:])

    ## correlate all the residuals, take the relevant values from the first row
    mass_pcorrs = np.corrcoef(res_by_z)[0, 1:]

    assert len(mass_pcorrs)==massY.shape[0]

    return(mass_pcorrs) 

def _debug_logging(c_band, r_band, valid_pixels, pif_pixels):
    ''' Optional logging information
    '''
    logging.debug('PIF: Original corrcoef = {}'.format(
        numpy.corrcoef(c_band[valid_pixels], r_band[valid_pixels])[0, 1]))

    if pif_pixels[0] != [] and pif_pixels[1] != []:
        logging.debug('PIF: Filtered corrcoef = {}'.format(
            numpy.corrcoef(c_band[pif_pixels], r_band[pif_pixels])[0, 1])) 

def apply_one(self, data, meta=None):
        return np.corrcoef(data) 

def learn_batch(self, X, Y, w=None, metrics=(), update=True):
        m = len(X)
        Y_correlations = np.corrcoef(Y.reshape((m, -1)))

        for layer in self.trainable_layers:
            h_correlations = np.corrcoef(layer.output.reshape(m, -1))
            local_error = mse(h_correlations.flat, Y_correlations.flat) 

def ncc(ypred, y):
    return np.corrcoef(ypred, y)[1,0] 

def ncc(ypred, y):
    return np.corrcoef(ypred, y)[1,0] 

def ncc(ypred, y):
    return np.corrcoef(ypred, y)[1,0] 

def scoremodel(model, x, y):
    '''Return fitness of model. We'll use R^2 and RMS.
       We also compare to the RMS with the mean.'''
    p = model.predict(x).squeeze()
    r = rms(p-y)
    aver = rms(y-np.mean(y))  #RMS if we just used average
    if np.std(p) == 0.0 or np.std(y) == 0: #R not defined
        return 0,r,aver
        
    return np.corrcoef(p,y)[0][1]**2,r,aver 

def scoremodel(model, x, y):
    '''Return fitness of model. We'll use R^2 and RMS.
       We also compare to the RMS with the mean.'''
    p = model.predict(x).squeeze()
    r = rms(p-y)
    aver = rms(y-np.mean(y))  #RMS if we just used average
    return np.corrcoef(p,y)[0,1]**2,r,aver