Python scipy.stats.linregress() Examples

def correlation_plot(x, y,
                     save_path,
                     title,
                     xlabel, ylabel):
    plt.scatter(x, y)
    slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
    line_x = np.arange(x.min(), x.max())
    line_y = slope*line_x + intercept
    plt.plot(line_x, line_y,
             label='$%.2fx + %.2f$, $R^2=%.2f$' % (slope, intercept, r_value**2))
    plt.legend(loc='best')
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.tight_layout()
    plt.savefig(save_path)
    plt.clf()  # clear figure
    plt.close() 

def fit(self, verbose=True):
        """
        fit exponential trying find annualized increase

        :param verbose: if True (default), print debug info of linear regression
        :return:
        """
        df = self.df
        slope_b, intercept_b, r_b, p_b, std_err_b = stats.linregress(
            df["date_count"], df["lnb"]
        )
        slope_e, intercept_e, r_e, p_e, std_err_e = stats.linregress(
            df["date_count"], df["lne"]
        )
        if verbose:
            print("B fit", slope_b, intercept_b, r_b, p_b, std_err_b)
            print("E fit", slope_e, intercept_e, r_e, p_e, std_err_e)
        self.slope_b = slope_b
        self.intercept_b = intercept_b
        self.slope_e = slope_e
        self.intercept_e = intercept_e 

def produce_the_kde_plot(cycles, color, save_name):
    ground_truth_and_suggested = [(eval_code.get_best_qed_from_smiles_bag(elem['ground_truth_product']),
                                   eval_code.get_best_qed_from_smiles_bag(elem['suggested_product']))
                                         for elem in cycles]
    len_out = len(ground_truth_and_suggested)
    ground_truth_and_suggested = [elem for elem in ground_truth_and_suggested if elem[1] != -np.inf]
    len_filter = len(ground_truth_and_suggested)
    num_discarding = len_out - len_filter
    if num_discarding:
        warnings.warn(f"Discarding {num_discarding} our of {len_out} as no successful reconstruction")
    ground_truth_and_suggested = np.array(ground_truth_and_suggested)
    ground_truth_product_qed = ground_truth_and_suggested[:, 0]
    suggested_product_qed = ground_truth_and_suggested[:, 1]

    g = sns.jointplot(x=ground_truth_product_qed, y=suggested_product_qed, kind="kde", color=color,
                      )
    g.set_axis_labels("product's QED", "reconstructed product's QED", fontsize=16)
    rsquare = lambda a, b: stats.pearsonr(ground_truth_product_qed, suggested_product_qed)[0] ** 2
    g = g.annotate(rsquare, template="{stat}: {val:.2f}",
                   stat="$R^2$", loc="upper left", fontsize=12)
    print(f"Rsquare: {stats.pearsonr(ground_truth_product_qed, suggested_product_qed)[0] ** 2}")
    print(f"scipystats: {stats.linregress(ground_truth_product_qed, suggested_product_qed)}")
    plt.tight_layout()
    plt.savefig(f"{save_name}.pdf") 

def test_nist_norris(self):
        x = [0.2, 337.4, 118.2, 884.6, 10.1, 226.5, 666.3, 996.3, 448.6, 777.0,
             558.2, 0.4, 0.6, 775.5, 666.9, 338.0, 447.5, 11.6, 556.0, 228.1,
             995.8, 887.6, 120.2, 0.3, 0.3, 556.8, 339.1, 887.2, 999.0, 779.0,
             11.1, 118.3, 229.2, 669.1, 448.9, 0.5]

        y = [0.1, 338.8, 118.1, 888.0, 9.2, 228.1, 668.5, 998.5, 449.1, 778.9,
             559.2, 0.3, 0.1, 778.1, 668.8, 339.3, 448.9, 10.8, 557.7, 228.3,
             998.0, 888.8, 119.6, 0.3, 0.6, 557.6, 339.3, 888.0, 998.5, 778.9,
             10.2, 117.6, 228.9, 668.4, 449.2, 0.2]

        # Expected values
        exp_slope = 1.00211681802045
        exp_intercept = -0.262323073774029
        exp_rvalue = 0.999993745883712

        actual = stats.linregress(x, y)

        assert_almost_equal(actual.slope, exp_slope)
        assert_almost_equal(actual.intercept, exp_intercept)
        assert_almost_equal(actual.rvalue, exp_rvalue, decimal=5) 

def calc_beta(stock, benchmark, price_history):
    """
    Calculate beta amounts for each security
    """
    stock_prices = price_history[stock].pct_change().dropna()
    bench_prices = price_history[benchmark].pct_change().dropna()
    aligned_prices = bench_prices.align(stock_prices, join='inner')
    bench_prices = aligned_prices[0]
    stock_prices = aligned_prices[1]
    bench_prices = np.array(bench_prices.values)
    stock_prices = np.array(stock_prices.values)
    bench_prices = np.reshape(bench_prices, len(bench_prices))
    stock_prices = np.reshape(stock_prices, len(stock_prices))
    if len(stock_prices) == 0:
        return None
    # market_beta, benchmark_beta = np.polyfit(bench_prices, stock_prices, 1)
    slope, intercept, r_value, p_value, stderr = stats.linregress(bench_prices, stock_prices)
    return slope 

def calc_beta_testing(stock, benchmark, price_history):
    """
    Calculate beta and alpha amounts for each security
    """
    # stock_prices = price_history[stock].pct_change().dropna()
    stock_prices = np.log(1+price_history[stock].pct_change().dropna())
    # bench_prices = price_history[benchmark].pct_change().dropna()
    bench_prices = np.log(1+price_history[benchmark].pct_change().dropna())
    aligned_prices = bench_prices.align(stock_prices, join='inner')
    bench_prices = aligned_prices[0]
    stock_prices = aligned_prices[1]
    bench_prices = np.array(bench_prices.values)
    stock_prices = np.array(stock_prices.values)
    bench_prices = np.reshape(bench_prices, len(bench_prices))
    stock_prices = np.reshape(stock_prices, len(stock_prices))

    if len(stock_prices) == 0:
        return None
    # market_beta, benchmark_beta = np.polyfit(bench_prices, stock_prices, 1)
    slope, intercept, r_value, p_value, stderr = stats.linregress(bench_prices, stock_prices)
    return slope, intercept 

def generate_ols_regression_pixel_list(candidate_pifs, reference_pifs):
    ''' Performs PCA analysis on the valid pixels and filters according
    to the distance from the principle eigenvector.

    :param list candidate_pifs: A list of candidate PIF data
    :param list reference_pifs: A list of coincident reference PIF data

    :returns: A LinearTransformation object (gain and offset)
    '''
    logging.info('Transformation: Calculating ordinary least squares '
                 'regression transformations')

    gain, offset, r_value, p_value, std_err = linregress(
        candidate_pifs, reference_pifs)
    logging.debug(
        'Fit statistics: r_value = {}, p_value = {}, std_err = {}'.format(
            r_value, p_value, std_err))
    logging.info("Transformation: gain {}, offset {}".format(gain, offset))

    return LinearTransformation(gain, offset) 

def test_regress_two_inputs_horizontal_line(self):
        # Regress a horizontal line formed by two points.
        x = np.arange(2)
        y = np.ones(2)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 1.0)  # horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 

def _guess_delay(self,f_data,z_data):
        phase2 = np.unwrap(np.angle(z_data))
        gradient, intercept, r_value, p_value, std_err = stats.linregress(f_data,phase2)
        return gradient*(-1.)/(np.pi*2.) 

def stability_of_timeseries(returns):
    """Determines R-squared of a linear fit to the cumulative
    log returns. Computes an ordinary least squares linear fit,
    and returns R-squared.

    Parameters
    ----------
    returns : pd.Series or np.ndarray
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~empyrical.stats.cum_returns`.

    Returns
    -------
    float
        R-squared.

    """
    if len(returns) < 2:
        return np.nan

    returns = np.asanyarray(returns)
    returns = returns[~np.isnan(returns)]

    cum_log_returns = np.log1p(returns).cumsum()
    rhat = stats.linregress(np.arange(len(cum_log_returns)),
                            cum_log_returns)[2]

    return rhat ** 2 

def test_empty_input(self):
        assert_raises(ValueError, stats.linregress, [], []) 

def test_nan_input(self):
        x = np.arange(10.)
        x[9] = np.nan

        with np.errstate(invalid="ignore"):
            assert_array_equal(stats.linregress(x, x),
                               (np.nan, np.nan, np.nan, np.nan, np.nan)) 

def test_expo_function():

    off, knee, exp = 10, 5, 2

    xs = np.arange(1, 100)
    ys = expo_function(xs, off, knee, exp)

    assert np.all(ys)

    # Note: no obvious way to test the knee specifically
    #  Here - test that past the knee, has expected exponent & offset value
    exp_meas, off_meas, _, _, _ = linregress(np.log10(xs[knee**2:]), ys[knee**2:])

    assert np.isclose(off_meas, off, 0.1)
    assert np.isclose(np.abs(exp_meas), exp, 0.1) 

def test_r2_score():
    x = np.linspace(0, 1, 100)
    y = np.random.normal(x, 1)
    res = linregress(x, y)
    assert_allclose(res.rvalue ** 2, r2_score(y, res.intercept + res.slope * x).r2, 2) 

def _calculate_subspace(self, S, f):
        parameters = [Parameter(distribution='uniform', lower=self.bounds_l[i], \
                upper=self.bounds_u[i], order=1) for i in range(0, self.n)]
        poly = Poly(parameters, basis=Basis('total-order'), method='least-squares', \
                sampling_args={'sample-points': S, 'sample-outputs': f})
        poly.set_model()
        Subs = Subspaces(full_space_poly=poly, method='active-subspace', subspace_dimension=self.d)
        U0 = Subs.get_subspace()[:,0].reshape(-1,1)
        U1 = Subs.get_subspace()[:,1:]
        for i in range(self.d-1):
            R = []
            for j in range(U1.shape[1]):
                U = np.hstack((U0, U1[:, j].reshape(-1,1)))
                Y = np.dot(S, U)
                myParameters = [Parameter(distribution='uniform', lower=np.min(Y[:,k]), upper=np.max(Y[:,k]), \
                        order=2) for k in range(Y.shape[1])]
                myBasis = Basis('total-order')
                poly = Poly(myParameters, myBasis, method='least-squares', \
                        sampling_args={'sample-points':Y, 'sample-outputs':f})
                poly.set_model()
                _,_,r,_,_ = linregress(poly.get_polyfit(Y).flatten(),f.flatten()) 
                R.append(r**2)
            index = np.argmax(R)
            U0 = np.hstack((U0, U1[:, index].reshape(-1,1)))
            U1 = np.delete(U1, index, 1)
        if self.subspace_method == 'variable-projection':
            Subs = Subspaces(method='variable-projection', sample_points=S, sample_outputs=f, \
                    subspace_init=U0, subspace_dimension=self.d, polynomial_degree=2, tol=1.0e-5, max_iter=2*self.d*self.n)
            self.U = Subs.get_subspace()[:, :self.d]
        elif self.subspace_method == 'active-subspaces':
            self.U = U0 

def detrend(x,y):
	result = stats.linregress(x,y)
	slope = result[0]
	intercept = result[1]
	detrended = []
	for val in range(len(y)):
		pred = slope*val+intercept
		residual = y[val]-pred
		detrended.append(residual)
	return (detrended,slope,intercept)

# Applies a linear trend to a series of data points
# i.e. a[z] = y[z] + trend
#           = y[z] + slope*x[z] + intercept 

def predictLinearRegression(xTrain,yTrain,xTest):
	result = stats.linregress(xTrain,yTrain)
	slope = result[0]
	intercept = result[1]
	predict = np.zeros(len(xTest))
	for x in range(len(xTest)):
		predict[x] = slope*xTest[x]+intercept
	return predict

# Performs linear regression on univariate time
# series repr. by xTrain,yTrain, and then makes predictions
# for xTest
# Returns a 1-d vector representing predictions 

def fit_lm(y):   
    x = np.array(range(len(y)))
    gradient, intercept, r_value, p_value, std_err = stats.linregress(x,y)
    fit_line = [(x_val * gradient) + intercept for x_val in x]
    return gradient, intercept, r_value, p_value, std_err, fit_line 

def test_regress_two_inputs(self):
        # Regress a simple line formed by two points.
        x = np.arange(2)
        y = np.arange(3, 5)

        res = stats.linregress(x, y)
        assert_almost_equal(res[3], 0.0)  # non-horizontal line
        assert_almost_equal(res[4], 0.0)  # zero stderr 

def test_regress_simple_negative_cor(self):
        # If the slope of the regression is negative the factor R tend to -1 not 1.
        # Sometimes rounding errors makes it < -1 leading to stderr being NaN
        a, n = 1e-71, 100000
        x = np.linspace(a, 2 * a, n)
        y = np.linspace(2 * a, a, n)
        stats.linregress(x, y)
        res = stats.linregress(x, y)
        assert_(res[2] >= -1)  # propagated numerical errors were not corrected
        assert_almost_equal(res[2], -1)  # perfect negative correlation case
        assert_(not np.isnan(res[4]))  # stderr should stay finite 

def test_linregress(self):
        # compared with multivariate ols with pinv
        x = np.arange(11)
        y = np.arange(5,16)
        y[[(1),(-2)]] -= 1
        y[[(0),(-1)]] += 1

        res = (1.0, 5.0, 0.98229948625750, 7.45259691e-008, 0.063564172616372733)
        assert_array_almost_equal(stats.linregress(x,y),res,decimal=14) 

def test_regress_shape_error(self):
        # Check that a single input argument to linregress with wrong shape
        # results in a ValueError.
        assert_raises(ValueError, stats.linregress, np.ones((3, 3))) 

def get_trend(books, trades):
    '''
    Returns the linear trend in previous trades for each data point in DataFrame
    of book data
    '''

    def trend(x):
        trades_n = trades.iloc[x.indexes[0]:x.indexes[1]]
        if len(trades_n) < 3:
            return 0
        else:
            return linregress(trades_n.index.values, trades_n.price.values)[0]
    return books.apply(trend, axis=1) 

def test_regress_simple_onearg_cols(self):
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        cols = np.hstack((np.expand_dims(x, 1), np.expand_dims(y, 1)))

        res = stats.linregress(cols)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 

def test_regress_simple_onearg_rows(self):
        # Regress a line w sinusoidal noise, with a single input of shape (2, N).
        x = np.linspace(0, 100, 100)
        y = 0.2 * np.linspace(0, 100, 100) + 10
        y += np.sin(np.linspace(0, 20, 100))
        rows = np.vstack((x, y))

        res = stats.linregress(rows)
        assert_almost_equal(res[4], 2.3957814497838803e-3) 

def test_regressZEROX(self):
        # W.IV.D. Regress ZERO on X.
        # The program should inform you that ZERO has no variance or it should
        # go ahead and compute the regression and report a correlation and
        # total sum of squares of exactly 0.
        y = stats.linregress(X,ZERO)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,0.0) 

def test_regressXX(self):
        # W.IV.B.  Regress X on X.
        # The constant should be exactly 0 and the regression coefficient should be 1.
        # This is a perfectly valid regression.  The program should not complain.
        y = stats.linregress(X,X)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,0.0)
        assert_almost_equal(r,1.0)
#     W.IV.C. Regress X on BIG and LITTLE (two predictors).  The program
#     should tell you that this model is "singular" because BIG and
#     LITTLE are linear combinations of each other.  Cryptic error
#     messages are unacceptable here.  Singularity is the most
#     fundamental regression error.
# Need to figure out how to handle multiple linear regression.  Not obvious 

def test_linregressBIGX(self):
        # W.II.F.  Regress BIG on X.
        # The constant should be 99999990 and the regression coefficient should be 1.
        y = stats.linregress(X,BIG)
        intercept = y[1]
        r = y[2]
        assert_almost_equal(intercept,99999990)
        assert_almost_equal(r,1.0) 

def test_linregress(self):
        for n in self.get_n():
            x, y, xm, ym = self.generate_xy_sample(n)
            res1 = stats.linregress(x, y)
            res2 = stats.mstats.linregress(xm, ym)
            assert_allclose(np.asarray(res1), np.asarray(res2)) 

def test_expo_nk_function():

    off, exp = 10, 2

    xs = np.arange(1, 100)
    ys = expo_nk_function(xs, off, exp)

    assert np.all(ys)

    # By design, this expo function assumes linear xs and log-space ys
    #   Where the log-log should be a straight line. Use that to test.
    sl_meas, off_meas, _, _, _ = linregress(np.log10(xs), ys)

    assert np.isclose(off, off_meas)
    assert np.isclose(exp, np.abs(sl_meas))