Python scipy.stats.linregress() Examples

The following are code examples for showing how to use scipy.stats.linregress(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: whiskyInvest   Author: arms3   File: fetch.py    MIT License 7 votes vote down vote up
def analyse_prices(pitches, df):
    """Perform linear regression analysis on all pitches individually."""
    grouped = df.groupby('pitchId')
    linreg = {}
    for key, grp in grouped:
        slope, intercept, r_value, p_value, std_err = stats.linregress(grp.day, grp.priceAvg)
        linreg[int(key)] = {'slope': slope, 'intercept': intercept, 'r_value': r_value}

    # Convert to df
    linreg = pd.DataFrame.from_dict(linreg, orient='index')

    # Calculate returns
    pitches.set_index('pitchId', inplace=True)
    pitches = pitches.join(linreg)
    pitches = calc_returns(pitches)

    # Clean up and return
    pitches.reset_index(inplace=True)
    return pitches 
Example 2
Project: radiometric_normalization   Author: planetlabs   File: transformation.py    Apache License 2.0 6 votes vote down vote up
def generate_ols_regression_pixel_list(candidate_pifs, reference_pifs):
    ''' Performs PCA analysis on the valid pixels and filters according
    to the distance from the principle eigenvector.

    :param list candidate_pifs: A list of candidate PIF data
    :param list reference_pifs: A list of coincident reference PIF data

    :returns: A LinearTransformation object (gain and offset)
    '''
    logging.info('Transformation: Calculating ordinary least squares '
                 'regression transformations')

    gain, offset, r_value, p_value, std_err = linregress(
        candidate_pifs, reference_pifs)
    logging.debug(
        'Fit statistics: r_value = {}, p_value = {}, std_err = {}'.format(
            r_value, p_value, std_err))
    logging.info("Transformation: gain {}, offset {}".format(gain, offset))

    return LinearTransformation(gain, offset) 
Example 3
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 6 votes vote down vote up
def test_nist_norris(self):
        x = [0.2, 337.4, 118.2, 884.6, 10.1, 226.5, 666.3, 996.3, 448.6, 777.0,
             558.2, 0.4, 0.6, 775.5, 666.9, 338.0, 447.5, 11.6, 556.0, 228.1,
             995.8, 887.6, 120.2, 0.3, 0.3, 556.8, 339.1, 887.2, 999.0, 779.0,
             11.1, 118.3, 229.2, 669.1, 448.9, 0.5]

        y = [0.1, 338.8, 118.1, 888.0, 9.2, 228.1, 668.5, 998.5, 449.1, 778.9,
             559.2, 0.3, 0.1, 778.1, 668.8, 339.3, 448.9, 10.8, 557.7, 228.3,
             998.0, 888.8, 119.6, 0.3, 0.6, 557.6, 339.3, 888.0, 998.5, 778.9,
             10.2, 117.6, 228.9, 668.4, 449.2, 0.2]

        # Expected values
        exp_slope = 1.00211681802045
        exp_intercept = -0.262323073774029
        exp_rvalue = 0.999993745883712

        actual = stats.linregress(x, y)

        assert_almost_equal(actual.slope, exp_slope)
        assert_almost_equal(actual.intercept, exp_intercept)
        assert_almost_equal(actual.rvalue, exp_rvalue, decimal=5) 
Example 4
Project: LSDMappingTools   Author: LSDtopotools   File: statsutilities.py    MIT License 6 votes vote down vote up
def linregress_residuals(xdata,ydata):
    """
    This function performs a linear regression and then gets the residuals
    
    Args:
        xdata (array-like): The x data
        ydata (array-like): The y data
        
    Returns:
        residuals: the residuals of the regression
        slope: the slope of regression line
        intercept: intercept of the regression line
        rvalue: correlation coeffficient
        pvalue: two-sided p-value for a hypothesis test whose null hypothesis is that the slope is zero.
        stderr: standard error of the estimated gradient
    
    Author: SMM
    """
    
    from scipy import stats
    
    # Get the regression
    (m,b,r,pvalue,stderr)=stats.linregress(xdata,ydata)   
    #print("In regress1, m is: "+str(m))    
    # get the residuals
    residuals = np.copy(xdata)
    for idx,x in enumerate(xdata):
        yfit = m*x+b
        residuals[idx] = yfit-ydata[idx]
        
    return (residuals,m,b,r,pvalue,stderr) 
Example 5
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 6 votes vote down vote up
def test_nist_norris(self):
        x = [0.2, 337.4, 118.2, 884.6, 10.1, 226.5, 666.3, 996.3, 448.6, 777.0,
             558.2, 0.4, 0.6, 775.5, 666.9, 338.0, 447.5, 11.6, 556.0, 228.1,
             995.8, 887.6, 120.2, 0.3, 0.3, 556.8, 339.1, 887.2, 999.0, 779.0,
             11.1, 118.3, 229.2, 669.1, 448.9, 0.5]

        y = [0.1, 338.8, 118.1, 888.0, 9.2, 228.1, 668.5, 998.5, 449.1, 778.9,
             559.2, 0.3, 0.1, 778.1, 668.8, 339.3, 448.9, 10.8, 557.7, 228.3,
             998.0, 888.8, 119.6, 0.3, 0.6, 557.6, 339.3, 888.0, 998.5, 778.9,
             10.2, 117.6, 228.9, 668.4, 449.2, 0.2]

        # Expected values
        exp_slope = 1.00211681802045
        exp_intercept = -0.262323073774029
        exp_rsquared = 0.999993745883712

        actual = stats.linregress(x, y)

        assert_almost_equal(actual.slope, exp_slope)
        assert_almost_equal(actual.intercept, exp_intercept)
        assert_almost_equal(actual.rvalue**2, exp_rsquared) 
Example 6
Project: PyRHEED   Author: yux1991   File: generate_report.py    MIT License 6 votes vote down vote up
def linear_fit(self):
        file_name = QtWidgets.QFileDialog.getSaveFileName(None,"choose save file name","./linear_fit_results.txt","TXT (*.txt)")
        if file_name:
            output = open(file_name[0],mode='w')
            for i in range(self.AZmin,self.AZmax+1):
                self.currentAzimuth = i
                Az = self.currentAzimuth*1.8+self.AzimuthStart
                F,K,Ferror = self.get_FK()
                F_part = F[10:len(F)-30]
                K_part = K[10:len(K)-30]
                slope, intercept, r, p, stderr = lrg(K_part*K_part,F_part*F_part)
                output.write('{:9.6f}\t{:9.6f}\t{:9.6f}\t{:9.6f}\n'.format(Az,slope,intercept,stderr))
                fig = plt.figure()
                ax = plt.subplot()
                ax.set_title(r'$h^{2} vs k^{2}$')
                ax.scatter(K_part*K_part,F_part*F_part)
                ax.plot(K_part*K_part,slope*K_part*K_part+intercept)
                ax.set_xlabel(r'$k^{2}$')
                ax.set_ylabel(r'$h^{2}$')
            output.close
            plt.show()
        self.update_log("The fitting is done!") 
Example 7
Project: gender_analysis   Author: dhmit   File: statistical.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def linear_regression(array1, array2, pvalue_target=0.05):
    """
    Perform a linear regression on two continuous variables that may or may not be correlated.
    Returns True if the correlation is significant, and false otherwise.

    :param array1: array-like, one set of continuous data to be compared to array2. e.g. list of publication years in a certain order of novels
    :param array2: array-like, second set of continuous data to be compared to array1, should be the same size as array1. e.g. he/she distance in the same order of novels as array1
    :param pvalue_target: largest p-value for which we consider the correlation statistically significant
    :return: True if the correlation is significant, False otherwise

    >>> a1 = np.array([1, 2, 3, 4, 5])
    >>> a2 = np.array([1, 2, 3, 4, 5])
    >>> linear_regression(a1, a2)
    True
    >>> a3 = np.array([3, 4, 8, 6, 2])
    >>> a4 = np.array([14, 8, 17, 9, 16])
    >>> linear_regression(a3, a4)
    False

    """

    pvalue = stats.linregress(array1, array2)[3]
    return pvalue < pvalue_target 
Example 8
Project: shepherd   Author: geissdoerfer   File: calibration.py    MIT License 5 votes vote down vote up
def from_measurements(cls, filename: Path):
        """Instantiates calibration data from calibration measurements.

        Args:
            filename (Path): Path to YAML formatted file containing calibration
                measurement values.
        
        Returns:
            CalibrationData object with extracted calibration data.
        """
        with open(filename, "r") as stream:
            calib_data = yaml.safe_load(stream)

        calib_dict = dict()

        for component in ["harvesting", "load", "emulation"]:
            calib_dict[component] = dict()
            for channel in ["voltage", "current"]:
                calib_dict[component][channel] = dict()
                sample_points = calib_data["measurements"][component][channel]
                x = np.empty(len(sample_points))
                y = np.empty(len(sample_points))
                for i, point in enumerate(sample_points):
                    x[i] = point["measured"]
                    y[i] = point["reference"]
                slope, intercept, _, _, _ = stats.linregress(x, y)
                calib_dict[component][channel]["gain"] = float(slope)
                calib_dict[component][channel]["offset"] = float(intercept)

        return cls(calib_dict) 
Example 9
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_regress_simple():
    # Regress a line with sinusoidal noise. Test for #1273.
    x = np.linspace(0, 100, 100)
    y = 0.2 * np.linspace(0, 100, 100) + 10
    y += np.sin(np.linspace(0, 20, 100))

    slope, intercept, r_value, p_value, sterr = mstats.linregress(x, y)
    assert_almost_equal(slope, 0.19644990055858422)
    assert_almost_equal(intercept, 10.211269918932341)

    # test for namedtuple attributes
    res = mstats.linregress(x, y)
    attributes = ('slope', 'intercept', 'rvalue', 'pvalue', 'stderr')
    check_named_results(res, attributes, ma=True) 
Example 10
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        for n in self.get_n():
            x, y, xm, ym = self.generate_xy_sample(n)
            res1 = stats.linregress(x, y)
            res2 = stats.mstats.linregress(xm, ym)
            assert_allclose(np.asarray(res1), np.asarray(res2)) 
Example 11
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 
Example 12
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
#     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
#     should tell you that this model is "singular" because BIG and
#     LITTLE are linear combinations of each other.  Cryptic error
#     messages are unacceptable here.  Singularity is the most
#     fundamental regression error.
# Need to figure out how to handle multiple linear regression.  Not obvious 
Example 13
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressZEROX(self):
        # W.IV.D. Regress ZERO on X.
        # The program should inform you that ZERO has no variance or it should
        # go ahead and compute the regression and report a correlation and
        # total sum of squares of exactly 0.
        y = stats.linregress(X,ZERO)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,0.0) 
Example 14
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_onearg_rows(self):
        # Regress a line w sinusoidal noise, with a single input of shape (2, N).
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        rows = np.vstack((x, y))

        res = stats.linregress(rows)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 15
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_onearg_cols(self):
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        cols = np.hstack((np.expand_dims(x, 1), np.expand_dims(y, 1)))

        res = stats.linregress(cols)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 16
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_shape_error(self):
        # Check that a single input argument to linregress with wrong shape
        # results in a ValueError.
        assert_raises(ValueError, stats.linregress, np.ones((3, 3))) 
Example 17
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        # compared with multivariate ols with pinv
        x = np.arange(11)
        y = np.arange(5,16)
        y[[(1),(-2)]] -= 1
        y[[(0),(-1)]] += 1

        res = (1.0, 5.0, 0.98229948625750, 7.45259691e-008, 0.063564172616372733)
        assert_array_almost_equal(stats.linregress(x,y),res,decimal=14) 
Example 18
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_negative_cor(self):
        # If the slope of the regression is negative the factor R tend to -1 not 1.
        # Sometimes rounding errors makes it < -1 leading to stderr being NaN
        a, n = 1e-71, 100000
        x = np.linspace(a, 2 * a, n)
        y = np.linspace(2 * a, a, n)
        stats.linregress(x, y)
        res = stats.linregress(x, y)
        assert_(res[2] >= -1)  # propagated numerical errors were not corrected
        assert_almost_equal(res[2], -1)  # perfect negative correlation case
        assert_(not np.isnan(res[4]))  # stderr should stay finite 
Example 19
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_two_inputs(self):
        # Regress a simple line formed by two points.
        x = np.arange(2)
        y = np.arange(3, 5)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 0.0)  # non-horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 
Example 20
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_two_inputs_horizontal_line(self):
        # Regress a horizontal line formed by two points.
        x = np.arange(2)
        y = np.ones(2)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 1.0)  # horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 
Example 21
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_empty_input(self):
        assert_raises(ValueError, stats.linregress, [], []) 
Example 22
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 5 votes vote down vote up
def test_nan_input(self):
        x = np.arange(10.)
        x[9] = np.nan

        with warnings.catch_warnings():
            warnings.simplefilter("ignore", RuntimeWarning)
            assert_array_equal(stats.linregress(x, x),
                               (np.nan, np.nan, np.nan, np.nan, np.nan)) 
Example 23
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regress_simple():
    # Regress a line with sinusoidal noise. Test for #1273.
    x = np.linspace(0, 100, 100)
    y = 0.2 * np.linspace(0, 100, 100) + 10
    y += np.sin(np.linspace(0, 20, 100))

    slope, intercept, r_value, p_value, sterr = mstats.linregress(x, y)
    assert_almost_equal(slope, 0.19644990055858422)
    assert_almost_equal(intercept, 10.211269918932341) 
Example 24
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_linregress(self):
        for n in self.get_n():
            x, y, xm, ym = self.generate_xy_sample(n)
            res1 = stats.linregress(x, y)
            res2 = stats.mstats.linregress(xm, ym)
            assert_allclose(np.asarray(res1), np.asarray(res2)) 
Example 25
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 
Example 26
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
##     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
##     should tell you that this model is "singular" because BIG and
##     LITTLE are linear combinations of each other.  Cryptic error
##     messages are unacceptable here.  Singularity is the most
##     fundamental regression error.
### Need to figure out how to handle multiple linear regression.  Not obvious 
Example 27
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regressZEROX(self):
        # W.IV.D. Regress ZERO on X.
        # The program should inform you that ZERO has no variance or it should
        # go ahead and compute the regression and report a correlation and
        # total sum of squares of exactly 0.
        y = stats.linregress(X,ZERO)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,0.0) 
Example 28
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regress_simple_onearg_rows(self):
        # Regress a line w sinusoidal noise, with a single input of shape (2, N).
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        rows = np.vstack((x, y))

        res = stats.linregress(rows)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 29
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regress_simple_onearg_cols(self):
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        cols = np.hstack((np.expand_dims(x, 1), np.expand_dims(y, 1)))

        res = stats.linregress(cols)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 30
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regress_shape_error(self):
        # Check that a single input argument to linregress with wrong shape
        # results in a ValueError.
        assert_raises(ValueError, stats.linregress, np.ones((3, 3))) 
Example 31
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_linregress(self):
        # compared with multivariate ols with pinv
        x = np.arange(11)
        y = np.arange(5,16)
        y[[(1),(-2)]] -= 1
        y[[(0),(-1)]] += 1

        res = (1.0, 5.0, 0.98229948625750, 7.45259691e-008, 0.063564172616372733)
        assert_array_almost_equal(stats.linregress(x,y),res,decimal=14) 
Example 32
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 5 votes vote down vote up
def test_regress_simple_negative_cor(self):
        # If the slope of the regression is negative the factor R tend to -1 not 1.
        # Sometimes rounding errors makes it < -1 leading to stderr being NaN
        a, n = 1e-71, 100000
        x = np.linspace(a, 2 * a, n)
        y = np.linspace(2 * a, a, n)
        stats.linregress(x, y)
        res = stats.linregress(x, y)
        assert_(res[2] >= -1)  # propagated numerical errors were not corrected
        assert_almost_equal(res[2], -1)  # perfect negative correlation case
        assert_(not np.isnan(res[4]))  # stderr should stay finite 
Example 33
Project: LSDMappingTools   Author: LSDtopotools   File: statsutilities.py    MIT License 5 votes vote down vote up
def remove_outlying_residuals(xdata,ydata,residuals):
    """
    This function removes data with outying residuals
    
    Args:
        xdata (array-like): The x data
        ydata (array-like): The y data    
        residuals: The residuals

        
    Returns:
        new_x: data with outlier removed
        new_y: data with outliers removed
        is_outlier_vec: A vec of bools denoiting if they are outliers
        m: the slope of the regression
        b: the intercept of the regression
            
    Author: SMM
    """
    from scipy import stats

    # get the outliers
    is_outlier_vec = is_outlier(residuals, thresh=3.5)
    
    # now remove the outliers from the dataset
    new_x = []
    new_y = []
    for idx,x in enumerate(xdata):
        if not is_outlier_vec[idx]:
            new_x.append(xdata[idx])
            new_y.append(ydata[idx])
  
    NX = np.asarray(new_x)
    NY = np.asarray(new_y)
    
    # now get the new regression
    (m,b,r,pvalue,stderr)=stats.linregress(NX,NY)

    return (NX,NY, is_outlier_vec, m,b) 
Example 34
Project: findr   Author: acic2015   File: linearReg.py    MIT License 5 votes vote down vote up
def main():
    timestamps = []
    bottom_norms = []
    top_norms = []

    # expects norms.dat in same directory. Can be changed to be a command-line arg
    f = open('norms.dat', 'r')
    for line in f:
        words = line.split(' ')
        timestamps.append(words[0][4:20])
        bottom_norms.append(words[1])
        top_norms.append(words[2])

    slope, intercept, r_value, p_value, std_err = stats.linregress(np.asarray(timestamps, float), np.asarray(bottom_norms, float))

    bottom_result = {
        'slope': slope,
        'intercept': intercept,
        'start_time': timestamps[0]
    }

    slope, intercept, r_value, p_value, std_err = stats.linregress(np.asarray(timestamps, float), np.asarray(top_norms, float))

    top_result = {
        'slope': slope,
        'intercept': intercept,
        'start_time': timestamps[0]
    }

    result = {
        "bottom": bottom_result,
        "top": top_result
    }

    return result 
Example 35
Project: TNT   Author: GaoangW   File: track_lib.py    GNU General Public License v3.0 5 votes vote down vote up
def linear_pred(y):
    if len(y)==1:
        return y
    else:
        x = np.array(range(0,len(y)))
        slope, intercept, _, _, _ = stats.linregress(x,y)
        return slope*len(y)+intercept 
Example 36
Project: TNT   Author: GaoangW   File: track_lib.py    GNU General Public License v3.0 5 votes vote down vote up
def linear_pred_v2(tr_t, tr_y, ts_t):
    ts_y = np.ones(len(ts_t))
    if len(tr_t)==1:
        ts_y = ts_y*tr_y
    else:
        slope, intercept, _, _, _ = stats.linregress(tr_t,tr_y)
        ts_y = slope*ts_t+intercept
    return ts_y 
Example 37
Project: TNT   Author: GaoangW   File: tracklet_utils_3d.py    GNU General Public License v3.0 5 votes vote down vote up
def linear_pred(y):
    if len(y)==1:
        return y
    else:
        x = np.array(range(0,len(y)))
        slope, intercept, _, _, _ = stats.linregress(x,y)
        return slope*len(y)+intercept 
Example 38
Project: TNT   Author: GaoangW   File: tracklet_utils_3d.py    GNU General Public License v3.0 5 votes vote down vote up
def linear_pred_v2(tr_t, tr_y, ts_t):
    ts_y = np.ones(len(ts_t))
    if len(tr_t)==1:
        ts_y = ts_y*tr_y
    else:
        slope, intercept, _, _, _ = stats.linregress(tr_t,tr_y)
        ts_y = slope*ts_t+intercept
    return ts_y 
Example 39
Project: TNT   Author: GaoangW   File: track_lib.py    GNU General Public License v3.0 5 votes vote down vote up
def linear_pred(y):
    if len(y)==1:
        return y
    else:
        x = np.array(range(0,len(y)))
        slope, intercept, _, _, _ = stats.linregress(x,y)
        return slope*len(y)+intercept 
Example 40
Project: Ossian   Author: candlewill   File: acoustic_stats.py    Apache License 2.0 5 votes vote down vote up
def fit_lm(y):   
    x = np.array(range(len(y)))
    gradient, intercept, r_value, p_value, std_err = stats.linregress(x,y)
    fit_line = [(x_val * gradient) + intercept for x_val in x]
    return gradient, intercept, r_value, p_value, std_err, fit_line 
Example 41
Project: group-anomaly-detection   Author: caisr-hh   File: transformer.py    MIT License 5 votes vote down vote up
def transform_slope(self, x):
        self.X = np.vstack([self.X, x])
        if len(self.X) < self.w: return np.zeros(x.shape)
        return np.array([ linregress(range(self.w), self.X[-self.w:, j])[0] for j in range(len(x)) ]) 
Example 42
Project: Deopen   Author: kimmo1019   File: Deopen_regression.py    MIT License 5 votes vote down vote up
def model_test(model, X_test, y_test,outputfile):
    #net.load_params_from('/path/to/weights_file')
    y_pred = model.predict(X_test)
    print stats.linregress(y_test,y_pred[:,0])
    hkl.dump([y_pred[:,0],y_test],outputfile)

    
    #save model parameters 
Example 43
Project: HN_SO_analysis   Author: dgwozdz   File: diff_nonstationary.py    MIT License 5 votes vote down vote up
def diff_nonstationary(x, alpha):
    
    """Returns number of differentiations required to transform
    a non-stationary time series into a stationary one. If 0 (zero) is
    returned, there's no need to differentiate."""
    """
    PARAMETERS:
    1) x - input time series
    2) alpha - significance level
    """
    
    i = 0 # no need to differentiate
    pvalue = adfuller(x, regression =
            ('ct' if
            stats.linregress( pd.Series(range(1, len(x)+1)), x ).pvalue<alpha
            else 'c')
            )[1]
    while pvalue>alpha:
        x = x.diff()
        pvalue = adfuller(x.dropna(),
            regression = 'c')[1]
        i += 1
        if pvalue<=alpha:
            break
    return(int(i))
    
### End of code 
Example 44
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_regress_simple():
    # Regress a line with sinusoidal noise. Test for #1273.
    x = np.linspace(0, 100, 100)
    y = 0.2 * np.linspace(0, 100, 100) + 10
    y += np.sin(np.linspace(0, 20, 100))

    slope, intercept, r_value, p_value, sterr = mstats.linregress(x, y)
    assert_almost_equal(slope, 0.19644990055858422)
    assert_almost_equal(intercept, 10.211269918932341)

    # test for namedtuple attributes
    res = mstats.linregress(x, y)
    attributes = ('slope', 'intercept', 'rvalue', 'pvalue', 'stderr')
    check_named_results(res, attributes, ma=True) 
Example 45
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_siegelslopes():
    # method should be exact for straight line
    y = 2 * np.arange(10) + 0.5
    assert_equal(mstats.siegelslopes(y), (2.0, 0.5))
    assert_equal(mstats.siegelslopes(y, method='separate'), (2.0, 0.5))

    x = 2 * np.arange(10)
    y = 5 * x - 3.0
    assert_equal(mstats.siegelslopes(y, x), (5.0, -3.0))
    assert_equal(mstats.siegelslopes(y, x, method='separate'), (5.0, -3.0))

    # method is robust to outliers: brekdown point of 50%
    y[:4] = 1000
    assert_equal(mstats.siegelslopes(y, x), (5.0, -3.0))

    # if there are no outliers, results should be comparble to linregress
    np.random.seed(231)
    x = np.arange(10)
    y = -2.3 + 0.3*x + stats.norm.rvs(size=10)
    slope_ols, intercept_ols, _, _, _ = stats.linregress(x, y)

    slope, intercept = mstats.siegelslopes(y, x)
    assert_allclose(slope, slope_ols, rtol=0.1)
    assert_allclose(intercept, intercept_ols, rtol=0.1)

    slope, intercept = mstats.siegelslopes(y, x, method='separate')
    assert_allclose(slope, slope_ols, rtol=0.1)
    assert_allclose(intercept, intercept_ols, rtol=0.1) 
Example 46
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        for n in self.get_n():
            x, y, xm, ym = self.generate_xy_sample(n)
            res1 = stats.linregress(x, y)
            res2 = stats.mstats.linregress(xm, ym)
            assert_allclose(np.asarray(res1), np.asarray(res2)) 
Example 47
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 
Example 48
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
#     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
#     should tell you that this model is "singular" because BIG and
#     LITTLE are linear combinations of each other.  Cryptic error
#     messages are unacceptable here.  Singularity is the most
#     fundamental regression error.
# Need to figure out how to handle multiple linear regression.  Not obvious 
Example 49
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple(self):
        # Regress a line with sinusoidal noise.
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))

        res = stats.linregress(x, y)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 50
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_onearg_rows(self):
        # Regress a line w sinusoidal noise, with a single input of shape (2, N).
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        rows = np.vstack((x, y))

        res = stats.linregress(rows)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 51
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_onearg_cols(self):
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        cols = np.hstack((np.expand_dims(x, 1), np.expand_dims(y, 1)))

        res = stats.linregress(cols)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 52
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_shape_error(self):
        # Check that a single input argument to linregress with wrong shape
        # results in a ValueError.
        assert_raises(ValueError, stats.linregress, np.ones((3, 3))) 
Example 53
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        # compared with multivariate ols with pinv
        x = np.arange(11)
        y = np.arange(5,16)
        y[[(1),(-2)]] -= 1
        y[[(0),(-1)]] += 1

        res = (1.0, 5.0, 0.98229948625750, 7.45259691e-008, 0.063564172616372733)
        assert_array_almost_equal(stats.linregress(x,y),res,decimal=14) 
Example 54
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregress_result_attributes(self):
        # Regress a line with sinusoidal noise.
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))

        res = stats.linregress(x, y)
        attributes = ('slope', 'intercept', 'rvalue', 'pvalue', 'stderr')
        check_named_results(res, attributes) 
Example 55
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_two_inputs(self):
        # Regress a simple line formed by two points.
        x = np.arange(2)
        y = np.arange(3, 5)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 0.0)  # non-horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 
Example 56
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_two_inputs_horizontal_line(self):
        # Regress a horizontal line formed by two points.
        x = np.arange(2)
        y = np.ones(2)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 1.0)  # horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 
Example 57
Project: ble5-nrf52-mac   Author: tomasero   File: test_stats.py    MIT License 5 votes vote down vote up
def test_empty_input(self):
        assert_raises(ValueError, stats.linregress, [], []) 
Example 58
Project: RLDonkeycar   Author: downingbots   File: polifit.py    MIT License 5 votes vote down vote up
def median(self, lines, prev_ms, prev_bs):
    if prev_ms is None:
        prev_ms = []
        prev_bs = []
    
    xs = []
    ys = []
    xs_med = []
    ys_med = []
    m = 0
    b = 0

    for line in lines:
        for x1, y1, x2, y2 in line:
            xs += [x1, x2]
            ys += [y1, y2]
    
    if len(xs) > 2 and len(ys) > 2:
#         m, b = np.polyfit(xs, ys, 1)
        m, b, r_value_left, p_value_left, std_err = linregress(xs, ys)

        if len(prev_ms) > 0:
            prev_ms.append(m)
            prev_bs.append(b)
        else:
            return np.poly1d([m, b])
    
    if len(prev_ms) > 0:
        return np.poly1d([np.average(prev_ms), np.average(prev_bs)])
    else:
        return None

  #############################
  # from https://github.com/dmytronasyrov/CarND-LaneLines-P1/blob/master/P1.ipynb 
Example 59
Project: seaice   Author: Image-Py   File: fit_tol_.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def lineregress(thick,gray):
    # data=np.array(list(thick.values()))
    slope, intercept, r_value, p_value, std_err = linregress(thick,gray)
    return slope,intercept 
Example 60
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 
Example 61
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
##     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
##     should tell you that this model is "singular" because BIG and
##     LITTLE are linear combinations of each other.  Cryptic error
##     messages are unacceptable here.  Singularity is the most
##     fundamental regression error.
### Need to figure out how to handle multiple linear regression.  Not obvious 
Example 62
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressZEROX(self):
        # W.IV.D. Regress ZERO on X.
        # The program should inform you that ZERO has no variance or it should
        # go ahead and compute the regression and report a correlation and
        # total sum of squares of exactly 0.
        y = stats.linregress(X,ZERO)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,0.0) 
Example 63
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple(self):
        # Regress a line with sinusoidal noise.
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))

        res = stats.linregress(x, y)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 64
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_onearg_rows(self):
        # Regress a line w sinusoidal noise, with a single input of shape (2, N).
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        rows = np.vstack((x, y))

        res = stats.linregress(rows)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 
Example 65
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_shape_error(self):
        # Check that a single input argument to linregress with wrong shape
        # results in a ValueError.
        assert_raises(ValueError, stats.linregress, np.ones((3, 3))) 
Example 66
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        # compared with multivariate ols with pinv
        x = np.arange(11)
        y = np.arange(5,16)
        y[[(1),(-2)]] -= 1
        y[[(0),(-1)]] += 1

        res = (1.0, 5.0, 0.98229948625750, 7.45259691e-008, 0.063564172616372733)
        assert_array_almost_equal(stats.linregress(x,y),res,decimal=14) 
Example 67
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regress_simple_negative_cor(self):
        # If the slope of the regression is negative the factor R tend to -1 not 1.
        # Sometimes rounding errors makes it < -1 leading to stderr being NaN
        a, n = 1e-71, 100000
        x = np.linspace(a, 2 * a, n)
        y = np.linspace(2 * a, a, n)
        stats.linregress(x, y)
        res = stats.linregress(x, y)
        assert_(res[2] >= -1)  # propagated numerical errors were not corrected
        assert_almost_equal(res[2], -1)  # perfect negative correlation case
        assert_(not np.isnan(res[4]))  # stderr should stay finite 
Example 68
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_regress_simple():
    # Regress a line with sinusoidal noise. Test for #1273.
    x = np.linspace(0, 100, 100)
    y = 0.2 * np.linspace(0, 100, 100) + 10
    y += np.sin(np.linspace(0, 20, 100))

    slope, intercept, r_value, p_value, sterr = mstats.linregress(x, y)
    assert_almost_equal(slope, 0.19644990055858422)
    assert_almost_equal(intercept, 10.211269918932341)

    # test for namedtuple attributes
    res = mstats.linregress(x, y)
    attributes = ('slope', 'intercept', 'rvalue', 'pvalue', 'stderr')
    check_named_results(res, attributes, ma=True) 
Example 69
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_linregress(self):
        for n in self.get_n():
            x, y, xm, ym = self.generate_xy_sample(n)
            res1 = stats.linregress(x, y)
            res2 = stats.mstats.linregress(xm, ym)
            assert_allclose(np.asarray(res1), np.asarray(res2)) 
Example 70
Project: poker   Author: surgebiswas   File: test_stats.py    MIT License 5 votes vote down vote up
def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 
Example 71
Project: poker   Author: surgebiswas   File: test_stats.py    MIT License 5 votes vote down vote up
def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
#     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
#     should tell you that this model is "singular" because BIG and
#     LITTLE are linear combinations of each other.  Cryptic error
#     messages are unacceptable here.  Singularity is the most
#     fundamental regression error.
# Need to figure out how to handle multiple linear regression.  Not obvious 
Example 72
Project: order   Author: ipudu   File: ionic.py    MIT License 4 votes vote down vote up
def calculate_conductivity(self):
        """ calculate the ionic conductivity """

        self.sigma = np.zeros(3)
        atom_names = self.reader.atom_names
        n_frames = self.reader.n_frames
        n_atoms = self.reader.n_atoms
        box = self.reader.box_size
        mask = self.mask
        coords = self.reader.coords
        delta_t = self.delta_t

        t = np.arange(float(n_frames)) * self.delta_t

        # calculate the area
        # area unit: Angstrom ^ 2
        area = np.zeros(n_frames-1)
        for i in range(n_frames-1):
            tmp = 1.
            for j in range(3):
                if j != self.d:
                    tmp *= box[i][j]
            area[i] = tmp

        average_area = np.average(area)

        # method 2 ########################################################

        self.I = self.total_intensity(atom_names, n_frames, n_atoms, 
                                 box, mask, coords, delta_t)

        t_ = t[:-1]
        slope, _, _, _, _ = stats.linregress(t_, self.I)
        #slope = np.average(self.I)
        
        # calculate the conductivity        
        # Q =  I * t
        # I unit: A
        # J unit: A / m ^2
        
        J = 1.602176e13 * slope / average_area

        self.sigma[1] = J / self.E
        
        # method 3 ########################################################
        self.Q = self.intensity_across_a_fixed_plane(atom_names, n_frames, 
                                                n_atoms, box, 
                                                mask, coords)

        slope, _, _, _, _ = stats.linregress(t,self.Q)

        # calculate the conductivity        
        # Q =  I * t
        # I unit: A
        # J unit: A / m ^2
        J = 1.602176e13 * slope / average_area

        # sigma unit: S / m
        self.sigma[2]  = J / self.E
        ##################################################################
        self.print_result() 
Example 73
Project: matplotlib_utilities   Author: dmccloskey   File: matplot.py    MIT License 4 votes vote down vote up
def scatterLinePlot(self,title_I,xlabel_I,ylabel_I,x_data_I,y_data_I,text_labels_I=[],fit_func_I='linear',show_eqn_I=True,show_r2_I=True,filename_I=None,show_plot_I=True):
        '''Create a scatter line plot and fitted line'''
        # Create the fit:
        if fit_func_I == 'linear':
            slope, intercept, r_value, p_value, std_err = linregress(x_data_I, y_data_I);
            r2 = r_value**2; #coefficient of determination
            x2 = x_data_I;
            y2 = [];
            for d in x2:
                y2.append(d*slope+intercept);
        elif fit_func_I=='lowess':
            #lowess
            x2 = numpy.array(x_data_I);
            y2_lowess = lowess.lowess(x2,numpy.array(y_data_I),f=0.1,iter=100)
            y2 = numpy.zeros_like(y2_lowess);
            for i,y2s in enumerate(y2_lowess):
                if i==0:
                    y2[i] = y2s;
                elif i!=0 and y2s<y2[i-1]:
                    y2[i] = y2[i-1];
                else:
                    y2[i] = y2s;
        # Create a Figure object.
        fig = plt.figure()
        # Create an Axes object.
        ax = fig.add_subplot(1,1,1) # one row, one column, first plot
        # Plot the data.
        ax.scatter(x_data_I, y_data_I, color="blue", marker="o")
        ax.plot(x2,y2,color='red',linestyle='-')
        # Add a title.
        ax.set_title(title_I)
        # Add some axis labels.
        ax.set_xlabel(xlabel_I)
        ax.set_ylabel(ylabel_I)
        # Label data points.
        if text_labels_I:
            for i, txt in enumerate(text_labels_I):
                ax.annotate(txt, (x_data_I[i],y_data_I[i]))
        # Show fit equation
        if show_eqn_I:
            fit_eqn = "y = " + str(slope) + "*x";
            if intercept < 0: fit_eqn += " " + str(intercept);
            elif intercept > 0: fit_eqn += " +" + str(intercept);
            ax.annotate(fit_eqn,(min(x_data_I),max(y_data_I)));
        # Show r2 value
        if show_r2_I:
            r2_label = "r2 = " + str(r2);
            ax.annotate(r2_label,(min(x_data_I),max(y_data_I)-0.5));
        # Show legend
        # Produce an image.
        if filename_I:
            fig.savefig(filename_I)
        # Show the image.
        if show_plot_I:
            plt.show(); 
Example 74
Project: DRCOG_Urbansim   Author: apdjustino   File: estimate_linear_regression_r.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def skip_test_estimation_one_var(self):
        """ Test a regression estimation for a model with one independent variable
        """
        
        # First, use scipy to get reference values to compare the results of our
        # R-based regression to.
        #print "using scipy to calculate reference regression..."
        # Example regression from: http://www2.warwick.ac.uk/fac/sci/moac/currentstudents/peter_cock/python/lin_reg/
        from scipy import stats
        x = [5.05, 6.75, 3.21, 2.66]
        y = [1.65, 26.5, -5.93, 7.96]
        gradient, intercept, r_value, p_value, std_err = stats.linregress(x,y)
        r_squared = r_value**2
        #print "Gradient and intercept", gradient, intercept
        ##Gradient and intercept 5.3935773612 -16.2811279931
        #print "R-squared", r_squared
        ##R-squared 0.524806275136
        #print "p-value", p_value
        ##p-value 0.275564857882
        
        # Next, setup the call to estimate_linear_regression_r.run(...)
        # Need to call run method on estimate_linear_regression_r, whose prototype is: 
        #   def run(self, data, regression=None, resources=None):
        #   regresion is not used by the run method
        #   things I need to store in resources:
        #     constant_position = resources.get("constant_position",  array([], dtype='int32')) #position for intercept
        #     coef_names = resources.get("coefficient_names",  nvar*[])
        #     outcome = resources["outcome"].astype("float64")
        
        # Create resources
        coeff = array(['EX'])
        resources = Resources()
        # No constant
        resources.add("constant_position", array([], dtype='int32'))
        resources.add("coefficient_names", coeff)
        resources.add("outcome", array(y))
        
        #data = array([x, y])
        data = resize(array([x]), (len(x), 1))
        
        # run RPy-based regression
        estimateR = estimate_linear_regression_r()
        result = estimateR.run(data, resources=resources)
        #print "results from RPy-base estimation: " + str(result)
    
        # Finally, compare the scipy-based regression to the R-based regression
        # Compare estimate of the independent 
        self.assertEqual(round(intercept, 4), round(result['estimators'][0], 4))
        # Compare the R-Squared
        self.assertEqual(round(r_squared, 6), round(result['other_info']['R-Squared'], 6)) 
Example 75
Project: LSDMappingTools   Author: LSDtopotools   File: LSDMap_SAPlotting.py    MIT License 4 votes vote down vote up
def LinearRegressionRawData(DataDirectory, DEM_prefix, basin_list=[],parallel=False):
    """
    This function performs a linear regression on all of the slope-area data.
    It returns a dataframe with the linear regression info for each basin key.

    Args:
        DataDirectory (str): the data directory
        DEM_prefix (str): the prefix of the DEM_prefix
        basin_list: a list of the basins to analyse, default = empty (all basins)

    Returns:
        pandas dataframe with the linear regression info

    Author: SMM and FJC
    """
    # read in binned data
    if not parallel:
      df = Helper.ReadRawSAData(DataDirectory, DEM_prefix)
    else:
      df = Helper.AppendRawSAData(DataDirectory, DEM_prefix)

    # get a list of the basins if needed
    if basin_list == []:
        print ("You didn't give me a basin list so I will analyse all the basins")
        basin_list = df['basin_key'].unique()

    # now do a linear regression for each basin
    columns = ['basin_key', 'regression_slope', 'std_err', 'R2', 'p_value']
    OutDF = pd.DataFrame(columns=columns)

    for basin_key in basin_list:
        df_slope = (df[df['basin_key']==basin_key]['slope']).values
        df_area = (df[df['basin_key']==basin_key]['drainage_area']).values

        logS = np.log10(df_slope[df_area!=0])
        logA = np.log10(df_area[df_area!=0])

        slope, intercept, r_value, p_value, std_err = stats.linregress(logA,logS)

        #print("Slope: " +str(slope)+ " std_err: "+str(std_err)+ " R2 is: " + str(r_value**2) + " p value is: " + str(p_value))
        this_key = int(basin_key)
        this_row = [this_key,abs(slope),std_err,r_value**2,p_value]
        OutDF.loc[basin_key] = this_row

    return OutDF

##=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
## Regressions
##=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= 
Example 76
Project: LSDMappingTools   Author: LSDtopotools   File: LSDMap_SAPlotting.py    MIT License 4 votes vote down vote up
def LinearRegressionRawDataByChannel(DataDirectory, DEM_prefix, basin_list=[]):
    """
    This function performs a linear regression on the raw slope-area data separated.
    by channel.
    It returns a dataframe with the linear regression info for each basin key.

    Args:
        DataDirectory (str): the data directory
        DEM_prefix (str): the prefix of the DEM_prefix
        basin_list: a list of the basins to analyse, default = empty (all basins)

    Returns:
        pandas dataframe with the linear regression info

    Author: SMM and FJC
    """
    # read in binned data


    df = Helper.ReadRawSAData(DataDirectory, DEM_prefix)

    # get a list of the basins if needed
    if basin_list == []:
        print ("You didn't give me a basin list so I will analyse all the basins")
        basin_list = df['basin_key'].unique()
        print (basin_list)


    # now do a linear regression for each basin
    columns = ['basin_key', 'source_key', 'regression_slope', 'std_err', 'R2', 'p_value']
    OutDF = pd.DataFrame(columns=columns)
    counter = 0

    for basin_key in basin_list:
        this_df = df[df['basin_key'] == basin_key]
        # get the sources for this basin
        these_sources = this_df['source_key'].unique()

        for source in these_sources:
            #mask the dataframe for this source
            df_slope = (df[df['source_key']==source]['slope']).values
            df_area = (df[df['source_key']==source]['drainage_area']).values

            logS = np.log10(df_slope[df_area!=0])
            logA = np.log10(df_area[df_area!=0])

            slope, intercept, r_value, p_value, std_err = stats.linregress(logA,logS)

            #print("Slope: " +str(slope)+ " std_err: "+str(std_err)+ " R2 is: " + str(r_value**2) + " p value is: " + str(p_value))
            this_basin_key = int(basin_key)
            this_source_key = int(source)
            this_row = [this_basin_key,this_source_key,slope,std_err,r_value**2,p_value]
            OutDF.loc[counter] = this_row
            counter+=1

    return OutDF 
Example 77
Project: LSDMappingTools   Author: LSDtopotools   File: LSDMap_SAPlotting.py    MIT License 4 votes vote down vote up
def LinearRegressionSegmentedData(DataDirectory, DEM_prefix, basin_list=[]):
    """
    This function performs a linear regression on each of the segments for the
    SA data.

    Args:
        DataDirectory (str): the data directory
        DEM_prefix (str): the prefix of the DEM_prefix
        basin_list: a list of the basins to analyse, default = empty (all basins)

    Returns:
        pandas dataframe with the linear regression info

    Author: FJC
    """

    df = Helper.ReadSegmentedSAData(DataDirectory, DEM_prefix)

    # get a list of the basins if needed
    if basin_list == []:
        print ("You didn't give me a basin list so I will analyse all the basins")
        basin_list = df['basin_key'].unique()

    columns = ['basin_key', 'segment_number', 'regression_slope', 'std_err', 'R2', 'p_value']
    OutDF = pd.DataFrame(columns=columns)
    counter=0

    # get the segments for each basin
    for basin_key in basin_list:
        print ("THIS BASIN IS: "+str(basin_key))
        SegmentDF = df[df['basin_key'] == basin_key]

        # get the data for each individual segment number
        segments = SegmentDF['segment_number'].unique()

        for segment_no in segments:
            print ("Segment number is: "+str(segment_no))
            ThisDF = SegmentDF[SegmentDF['segment_number']==segment_no]
            #now regress the data for this segment to get the best fit m/n
            median_log_S = ThisDF['median_log_S']
            median_log_A = ThisDF['median_log_A']
            slope, intercept, r_value, p_value, std_err = stats.linregress(median_log_A,median_log_S)
            print("Slope: " +str(slope)+ " std_err: "+str(std_err)+ " R2 is: " + str(r_value**2) + " p value is: " + str(p_value) + " intercept is: " +str(intercept))

            this_key = int(basin_key)
            this_row = [this_key,int(segment_no),slope,std_err,r_value**2,p_value]
            OutDF.loc[counter] = this_row
            counter+=1

    return OutDF 
Example 78
Project: LSDMappingTools   Author: LSDtopotools   File: LSDMap_HillslopeMorphology.py    MIT License 4 votes vote down vote up
def PlotDdLh(DataDirectory, FilenamePrefix, PlotDirectory):
    """
    Plot drainage density vs hillslope length
    FJC
    """

    df = pd.read_csv(PlotDirectory+FilenamePrefix+'_basin_hillslope_data.csv')

    #choose colormap
    ColourMap = cm.RdYlBu

    # get the basins
    basins = df['basin_keys'].unique()

    # set up the figure
    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(6,5))
    #ax = ax.ravel()
    # make a big subplot to allow sharing of axis labels
    #fig.add_subplot(111, frameon=False)
    # hide tick and tick label of the big axes
    #plt.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off')

    plt.scatter(df['Lh_median'], df['dd_median'], c=basins, cmap=ColourMap, edgecolors='k', zorder=100, s=50)
    plt.errorbar(df.Lh_median,df.dd_median,xerr=[df['Lh_lower_err'], df['Lh_upper_err']], yerr=[df['dd_lower_err'], df['dd_upper_err']],fmt='o', zorder=1, ecolor='0.5',markersize=1,mfc='white',mec='k')
    plt.xlabel('Hillslope length (m)')
    plt.ylabel('Drainage density (m/km$^2$)')
    #plt.xlim(100, 500)
    #plt.ylim(3500, 10000)

    # linregress
    slope, intercept, r_value, p_value, std_err = stats.linregress(np.log10(df['Lh_median']),np.log10(df['dd_median']))
    print(slope, 10**intercept, r_value, p_value)
    ax.text(0.5,0.9, '$D_d$ = {0:.{0}f}$L_H^{{{1:f}}}$'.format(10**intercept, round(slope,2), 2)+'\n p = '+str(round(p_value, 2)), transform=ax.transAxes)

    # add colour bar
    m = cm.ScalarMappable(cmap=ColourMap)
    m.set_array(basins)
    cbar = plt.colorbar(m)
    #tick_locator = ticker.MaxNLocator(nbins=5)
    #cbar.locator = tick_locator
    #cbar.update_ticks()
    cbar.ax.invert_yaxis()
    cbar.set_label('Basin ID')

    #save output
    plt.subplots_adjust(left=0.2)
    plt.savefig(PlotDirectory+FilenamePrefix +"_dd_lh.png", dpi=300)
    plt.clf() 
Example 79
Project: deepGroupv2   Author: albu5   File: utils.py    MIT License 4 votes vote down vote up
def pairwise_features_at_time_t(anno, t, del_t=10):
    # returns 2D list of pairwise features at time t in anno
    # anno is matrix annotation saved in csv_anno folder
    # del_t is block length on which analysis is to be done,should be generally fixed to 10 frames
    n_people = anno.shape[2]

    # initialize by empty features
    feature = np.zeros(shape=[2 * (8 + pose_max - pose_min + 1 + action_max - action_min + 1)])
    pairs = [[feature for i in range(n_people)] for j in range(n_people)]

    for i in range(n_people):
        for j in range(n_people):
            ped_i = np.squeeze(anno[t:t + del_t, :, i])
            ped_j = np.squeeze(anno[t:t + del_t, :, j])
            # print(ped_i.shape, ped_j.shape)

            if len(ped_i.shape) == 1:
                ped_i = np.expand_dims(ped_i, axis=0)
                ped_j = np.expand_dims(ped_j, axis=0)

            bbs_i = ped_i[:, 0:4]
            bbs_j = ped_j[:, 0:4]

            # if pose is 0 then it's an invalid or inactive person in scene
            valid_i = not(np.all(ped_i[:, 4] == 0))
            valid_j = not(np.all(ped_j[:, 4] == 0))

            # compute velocity and mean position from 10 frame data
            # and median poses and action to filter noisy observations
            if valid_i and valid_j:
                _del_t = bbs_j.shape[0]
                vx_i, mx_i, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_i[:, 0]))
                vy_i, my_i, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_i[:, 1]))
                vw_i, mw_i, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_i[:, 2]))
                vh_i, mh_i, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_i[:, 3]))
                vx_j, mx_j, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_j[:, 0]))
                vy_j, my_j, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_j[:, 1]))
                vw_j, mw_j, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_j[:, 2]))
                vh_j, mh_j, _, _, _ = linregress(np.arange(_del_t), np.squeeze(bbs_j[:, 3]))
                pose_i = np.median(a=one_hot(ped_i[:, 4], x_min=pose_min, x_max=pose_max), axis=0)
                pose_j = np.median(a=one_hot(ped_j[:, 4], x_min=pose_min, x_max=pose_max), axis=0)
                action_i = np.median(a=one_hot(ped_i[:, 5], x_min=action_min, x_max=action_max), axis=0)
                action_j = np.median(a=one_hot(ped_j[:, 5], x_min=action_min, x_max=action_max), axis=0)
                feature = np.hstack([np.array([vx_i, vy_i, vw_i, vh_i, mx_i, my_i, mw_i, mh_i]), pose_i, action_i,
                                     np.array([vx_j, vy_j, vw_j, vh_j, mx_j, my_j, mw_j, mh_j]), pose_j, action_j])
            else:
                # fill with empty features if any person is not in scene at time t
                feature = np.zeros(shape=[2 * (8 + pose_max - pose_min + 1 + action_max - action_min + 1)])
            pairs[i][j] = feature
    return pairs 
Example 80
Project: vnpy_crypto   Author: birforce   File: ros.py    MIT License 4 votes vote down vote up
def _impute(df, observations, censorship, transform_in, transform_out):
    """
    Executes the basic regression on order stat (ROS) proceedure.

    Uses ROS to impute censored from the best-fit line of a
    probability plot of the uncensored values.

    Parameters
    ----------
    df : pandas.DataFrame
    observations : str
        Name of the column in the dataframe that contains observed
        values. Censored values should be set to the detection (upper)
        limit.
    censorship : str
        Name of the column in the dataframe that indicates that a
        observation is left-censored. (i.e., True -> censored,
        False -> uncensored)
    transform_in, transform_out : callable
        Transformations to be applied to the data prior to fitting
        the line and after estimated values from that line. Typically,
        `numpy.log` and `numpy.exp` are used, respectively.

    Returns
    -------
    estimated : pandas.DataFrame
        A new dataframe with two new columns: "estimated" and "final".
        The "estimated" column contains of the values inferred from the
        best-fit line. The "final" column contains the estimated values
        only where the original observations were censored, and the original
        observations everwhere else.

    """

    # detect/non-detect selectors
    uncensored_mask = df[censorship] == False
    censored_mask = df[censorship] == True

    # fit a line to the logs of the detected data
    fit_params = stats.linregress(
        df['Zprelim'][uncensored_mask],
        transform_in(df[observations][uncensored_mask])
    )

    # pull out the slope and intercept for use later
    slope, intercept = fit_params[:2]

    # model the data based on the best-fit curve
    # (editted for pandas 0.14 compatibility; see commit 63f162e
    #  when `pipe` and `assign` are available)
    df.loc[:, 'estimated'] = transform_out(slope * df['Zprelim'][censored_mask] + intercept)
    df.loc[:, 'final'] = numpy.where(df[censorship], df['estimated'], df[observations])

    return df