Python scipy.stats.normaltest() Examples

The following are code examples for showing how to use scipy.stats.normaltest(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: GeoPy   Author: aerler   File: stats.py    GNU General Public License v3.0 6 votes vote down vote up
def kstest_wrapper(data, dist='norm', ignoreNaN=True, args=None, N=20, alternative='two-sided', mode='approx'):
  ''' Kolmogorov-Smirnov Test, to test whether or not the data is from a given distribution. The 
      returned p-value indicates the probability that the data is from the given distribution, 
      i.e. a low p-value means the data are likely not from the tested distribution.
      Note that, for this test, it is necessary to specify shape, location, and scale parameters,
      to obtain meaningful results (c,loc,scale). '''
  if ignoreNaN: 
    nonans = np.invert(np.isnan(data)) # test for NaN's
    if np.sum(nonans) < 3: return np.NaN # return, if less than 3 non-NaN's
    data = data[nonans] # remove NaN's
  if args is None: args = ()
  D, pval = ss.kstest(data, dist, args=args, N=N, alternative=alternative, mode=mode); del D
  return pval

# wrapper for normaltest, a SciPy function to test normality
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.normaltest.html 
Example 2
Project: GeoPy   Author: aerler   File: stats.py    GNU General Public License v3.0 6 votes vote down vote up
def normaltest_wrapper(data, axis=None, ignoreNaN=True):
  ''' SciPy test, to test whether or not the data is from a normal distribution. The 
      returned p-value indicates the probability that the data is from a normal distribution, 
      i.e. a low p-value means the data are likely not from a normal distribution.
      This is a combination of the skewtest and the kurtosistest and can be applied along a 
      specified axis of a multi-dimensional arrays (using the 'axis' keyword), or over the 
      flattened array (axis=None). '''
  if ignoreNaN:
    if axis is not None: raise DataError("NaN removal does not work reliably with ND-arrays.")
    nonans = np.invert(np.isnan(data)) # test for NaN's
    if np.sum(nonans) < 3: return np.NaN # return, if less than 3 non-NaN's
    data = data[nonans] # remove NaN's
  # N.B.: unfortunately this is still the best way to handle this...
  #       'propagate' will return NaN for the entire array if any NaN are present,
  #       'omit' will fail if any axis has less than 8 non-NaN values
  # nan_policy = 'omit' if ignoreNaN else 'propagate'
  k2, pval = ss.normaltest(data, axis=axis, nan_policy='propagate'); del k2
  return pval

# global variable that is used to retain parameters for Shapiro-wilk test 
Example 3
Project: GeoPy   Author: aerler   File: stats.py    GNU General Public License v3.0 6 votes vote down vote up
def rv_stats_test(data_array, nparams=0, dist_type=None, stats_test=None, ignoreNaN=True, N=20, 
              alternative='two-sided', mode='approx', reta=False):
  ''' perform a goodnes of fit test; handles masked/NaN values; defaults to Shapiro-Wilk for Normal 
      distributions and Kolmogorov-Smirnov test for all others '''
  if np.any(np.isnan(data_array[:nparams])): pval = np.NaN # sort of as a masked values
  else:
    if stats_test is None:
      stats_test = 'sw' if dist_type == 'norm' else 'ks'
    else: stats_test = stats_test.lower()
    if stats_test in ('ks','kstest'): 
      pval = kstest_wrapper(data_array[nparams:], dist=dist_type, args=data_array[:nparams], ignoreNaN=ignoreNaN, N=N, alternative=alternative, mode=mode)
    elif stats_test in ('normtest','normaltest'): 
      pval = normaltest_wrapper(data_array[nparams:], ignoreNaN=ignoreNaN)
    elif stats_test in ('sw','shapiro','shapirowilk'): 
      pval = shapiro_wrapper(data_array[nparams:], ignoreNaN=ignoreNaN)    
    else: raise ArgumentError(stats_test)
  return pval

# Subclass of DistVar implementing various random variable distributions 
Example 4
Project: vnpy_crypto   Author: birforce   File: stattools.py    MIT License 6 votes vote down vote up
def omni_normtest(resids, axis=0):
    """
    Omnibus test for normality

    Parameters
    -----------
    resid : array-like
    axis : int, optional
        Default is 0

    Returns
    -------
    Chi^2 score, two-tail probability
    """
    #TODO: change to exception in summary branch and catch in summary()
    #behavior changed between scipy 0.9 and 0.10
    resids = np.asarray(resids)
    n = resids.shape[axis]
    if n < 8:
        from warnings import warn
        warn("omni_normtest is not valid with less than 8 observations; %i "
             "samples were given." % int(n), ValueWarning)
        return np.nan, np.nan

    return stats.normaltest(resids, axis=axis) 
Example 5
Project: DIVE-backend   Author: MacroConnections   File: field_properties.py    GNU General Public License v3.0 6 votes vote down vote up
def get_normality(field_name, field_values, field_type, general_type, scale):
    normality = None
    if general_type is 'q':
        try:
            d = field_values.astype(np.float)
            normality_test_result = sc_stats.normaltest(d)
            if normality_test_result:
                statistic = normality_test_result.statistic
                pvalue = normality_test_result.pvalue
                if pvalue < 0.05:
                    normality = True
                else:
                    normality = False
        except ValueError:
            normality = None
    return normality 
Example 6
Project: mljar-supervised   Author: mljar   File: preprocessing_utils.py    MIT License 6 votes vote down vote up
def is_log_scale_needed(x_org):
        x = x_org[~pd.isnull(x_org)]
        # first scale on raw data
        x = preprocessing.scale(x)
        # second scale on log data
        x_log = preprocessing.scale(np.log(x - np.min(x) + 1))

        # the old approach, let's check how new approach will work
        # original_skew = np.abs(stats.skew(x))
        # log_skew = np.abs(stats.skew(x_log))
        # return log_skew < original_skew
        ########################################################################
        # p is probability of being normal distributions
        k2, p1 = stats.normaltest(x)
        k2, p2 = stats.normaltest(x_log)

        return p2 > p1 
Example 7
Project: cpnest   Author: johnveitch   File: test_1d.py    MIT License 6 votes vote down vote up
def test_evidence(self):
        # 2 sigma tolerance
        tolerance = 2.0*np.sqrt(self.work.NS.state.info/self.work.NS.Nlive)
        print('2-sigma statistic error in logZ: {0:0.3f}'.format(tolerance))
        print('Analytic logZ {0}'.format(self.model.analytic_log_Z))
        print('Estimated logZ {0}'.format(self.work.NS.logZ))
        pos=self.work.posterior_samples['x']
        #t,pval=stats.kstest(pos,self.model.distr.cdf)
        stat,pval = stats.normaltest(pos.T)
        print('Normal test p-value {0}'.format(str(pval)))
        plt.figure()
        plt.hist(pos.ravel(),normed=True)
        x=np.linspace(self.model.bounds[0][0],self.model.bounds[0][1],100)
        plt.plot(x,self.model.distr.pdf(x))
        plt.title('NormalTest pval = {0}'.format(pval))
        plt.savefig('posterior.png')
        plt.figure()
        plt.plot(pos.ravel(),',')
        plt.title('chain')
        plt.savefig('chain.png')
        self.assertTrue(np.abs(self.work.NS.logZ - GaussianModel.analytic_log_Z)<tolerance, 'Incorrect evidence for normalised distribution: {0:.3f} instead of {1:.3f}'.format(self.work.NS.logZ,GaussianModel.analytic_log_Z ))
        self.assertTrue(pval>0.01,'Normaltest test failed: KS stat = {0}'.format(pval)) 
Example 8
Project: tableone   Author: tompollard   File: tableone.py    MIT License 6 votes vote down vote up
def _normaltest(self, x):
        """
        Compute test for normal distribution.

        Null hypothesis: x comes from a normal distribution
        p < alpha suggests the null hypothesis can be rejected.
        """
        if len(x.values[~np.isnan(x.values)]) > 10:
            stat, p = stats.normaltest(x.values, nan_policy='omit')
        else:
            p = None
        # dropna=False argument in pivot_table does not function as expected
        # return -1 instead of None
        if pd.isnull(p):
            return -1
        return p 
Example 9
Project: Splunking-Crime   Author: nccgroup   File: stattools.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def omni_normtest(resids, axis=0):
    """
    Omnibus test for normality

    Parameters
    -----------
    resid : array-like
    axis : int, optional
        Default is 0

    Returns
    -------
    Chi^2 score, two-tail probability
    """
    #TODO: change to exception in summary branch and catch in summary()
    #behavior changed between scipy 0.9 and 0.10
    resids = np.asarray(resids)
    n = resids.shape[axis]
    if n < 8:
        from warnings import warn
        warn("omni_normtest is not valid with less than 8 observations; %i "
             "samples were given." % int(n), ValueWarning)
        return np.nan, np.nan

    return stats.normaltest(resids, axis=axis) 
Example 10
Project: physalia   Author: TQRG   File: analytics.py    MIT License 6 votes vote down vote up
def samples_are_normal(*samples, **options):
    """Test whether each sample differs from a normal distribution.

    Use both Shapiro-Wilk test and D'Agostino and Pearson's test
    to test the null hypothesis that the sample is drawn from a
    normal distribution.

    Returns:
        List of tuples (is_normal(bool),(statistic(float),pvalue(float)).

    """
    alpha = options.get('alpha', 0.05)
    results = []
    for sample in samples:
        (_, shapiro_pvalue) = shapiro_result = shapiro(sample)
        (_, normaltest_pvalue) = normaltest_result = normaltest(sample)
        results.append((
            not (normaltest_pvalue < alpha and shapiro_pvalue < alpha),
            shapiro_result,
            normaltest_result
        ))
    return results 
Example 11
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 12
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 13
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2,-1,0,1,2,3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 14
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 15
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 16
Project: QFiPy   Author: kouzapo   File: equities.py    MIT License 5 votes vote down vote up
def test_normality(self):
		"""
		This method returns the t-statistic and the p-value of the normality test of the
		asset's returns. 

		Returns:
		-------
			results: ndarray, a numpy array of the normality test results.
		"""

		results = stats.normaltest(self.calc_log_returns())

		return results 
Example 17
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 18
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 19
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2,-1,0,1,2,3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 20
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2)


#TODO: for all ttest functions, add tests with masked array inputs 
Example 21
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_normaltest(self):
        np.seterr(over='raise')
        for n in self.get_n():
            if n > 8:
                with warnings.catch_warnings():
                    warnings.filterwarnings('ignore', category=UserWarning)
                    x, y, xm, ym = self.generate_xy_sample(n)
                    r = stats.normaltest(x)
                    rm = stats.mstats.normaltest(xm)
                    assert_allclose(np.asarray(r), np.asarray(rm)) 
Example 22
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 23
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 24
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 25
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 26
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 27
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 5 votes vote down vote up
def test_normalitytests():
    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502, 0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    yield assert_array_almost_equal, stats.normaltest(x), (st_normal, pv_normal)
    yield assert_array_almost_equal, stats.skewtest(x), (st_skew, pv_skew)
    yield assert_array_almost_equal, stats.kurtosistest(x), (st_kurt, pv_kurt) 
Example 28
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 29
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 30
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2,-1,0,1,2,3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 31
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 32
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 33
Project: Data-Analysis   Author: amrosado   File: bfpDataParsing.py    MIT License 5 votes vote down vote up
def plusMinusForceNormalTest(self, plusMinusIndex, timeArray, forceArray):
        statistics = []
        pValues = []

        for i in range(0, len(timeArray)):
            if (i >= plusMinusIndex/2) and (i < (len(timeArray)-(plusMinusIndex/2))):
                normalTestForTime = scistats.normaltest(forceArray[i-plusMinusIndex/2:i+plusMinusIndex/2])
                statistics.append(normalTestForTime.statistic)
                pValues.append(normalTestForTime.pvalue)
        statisticArray = np.array(statistics)
        pValueArray = np.array(pValues)
        timeNArray = np.array(timeArray[plusMinusIndex:len(timeArray)])
        normalTestDict = {'statistics': statisticArray, 'time': timeNArray, 'pValues': pValueArray}
        return normalTestDict 
Example 34
Project: pl2mind   Author: MRN-Code   File: ols.py    GNU General Public License v2.0 5 votes vote down vote up
def omni(self):
        """
        Omnibus test for normality
        """
        return stats.normaltest(self.e) 
Example 35
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 36
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 37
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 38
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 39
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 40
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 41
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 42
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2,-1,0,1,2,3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 43
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 44
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 45
Project: drifter_ml   Author: EricSchles   File: columnar_tests.py    MIT License 5 votes vote down vote up
def is_normal(self, column):
        new_data_result = stats.normaltest(self.new_data[column])
        historical_data_result = stats.normaltest(self.historical_data[column])
        if new_data_result.pvalue > 0.05 and historical_data_result.pvalue > 0.05:
            return True
        return False 
Example 46
Project: DIVE-backend   Author: MacroConnections   File: score_specs.py    GNU General Public License v3.0 5 votes vote down vote up
def _normaltest(v):
    return normaltest(v).pvalue 
Example 47
Project: DIVE-backend   Author: MacroConnections   File: utilities.py    GNU General Public License v3.0 5 votes vote down vote up
def sets_normal(THRESHOLD, *args):
    '''
    If normalP is less than threshold, not considered normal
    '''
    normal = True;
    for arg in args:
        if len(arg) < 8:
            return False
        if stats.normaltest(arg)[1] < THRESHOLD:
            normal = False;

    return normal 
Example 48
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_vs_nonmasked(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_array_almost_equal(mstats.normaltest(x),
                                  stats.normaltest(x))
        assert_array_almost_equal(mstats.skewtest(x),
                                  stats.skewtest(x))
        assert_array_almost_equal(mstats.kurtosistest(x),
                                  stats.kurtosistest(x))

        funcs = [stats.normaltest, stats.skewtest, stats.kurtosistest]
        mfuncs = [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]
        x = [1, 2, 3, 4]
        for func, mfunc in zip(funcs, mfuncs):
            assert_raises(ValueError, func, x)
            assert_raises(ValueError, mfunc, x) 
Example 49
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_axis_None(self):
        # Test axis=None (equal to axis=0 for 1-D input)
        x = np.array((-2,-1,0,1,2,3)*4)**2
        assert_allclose(mstats.normaltest(x, axis=None), mstats.normaltest(x))
        assert_allclose(mstats.skewtest(x, axis=None), mstats.skewtest(x))
        assert_allclose(mstats.kurtosistest(x, axis=None),
                        mstats.kurtosistest(x)) 
Example 50
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_maskedarray_input(self):
        # Add some masked values, test result doesn't change
        x = np.array((-2,-1,0,1,2,3)*4)**2
        xm = np.ma.array(np.r_[np.inf, x, 10],
                         mask=np.r_[True, [False] * x.size, True])
        assert_allclose(mstats.normaltest(xm), stats.normaltest(x))
        assert_allclose(mstats.skewtest(xm), stats.skewtest(x))
        assert_allclose(mstats.kurtosistest(xm), stats.kurtosistest(x)) 
Example 51
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_nd_input(self):
        x = np.array((-2,-1,0,1,2,3)*4)**2
        x_2d = np.vstack([x] * 2).T
        for func in [mstats.normaltest, mstats.skewtest, mstats.kurtosistest]:
            res_1d = func(x)
            res_2d = func(x_2d)
            assert_allclose(res_2d[0], [res_1d[0]] * 2)
            assert_allclose(res_2d[1], [res_1d[1]] * 2) 
Example 52
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_normaltest_result_attributes(self):
        x = np.array((-2, -1, 0, 1, 2, 3)*4)**2
        res = mstats.normaltest(x)
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 53
Project: pregel   Author: shagunsodhani   File: ptest.py    MIT License 5 votes vote down vote up
def _check_normality(dist):
    '''
    Method to check if the samples in dist differs from a normal distribution.
    Return true if the dist is likely to be gaussian.
    '''
    _, pvalue = normaltest(dist)
    if (pvalue > 0.05):
        # The samples in dist came from a normal distribution with 95% confidence
        return True
    else:
        return False 
Example 54
Project: tableone   Author: tompollard   File: tableone.py    MIT License 5 votes vote down vote up
def _generate_remark_str(self, end_of_line='\n'):
        """
        Generate a series of remarks that the user should consider
        when interpreting the summary statistics.
        """
        warnings = {}
        msg = '{}'.format(end_of_line)

        # generate warnings for continuous variables
        if self._continuous:
            # highlight far outliers
            outlier_mask = self.cont_describe.far_outliers > 1
            outlier_vars = list(self.cont_describe.far_outliers[outlier_mask].dropna(how='all').index)
            if outlier_vars:
                warnings["Warning, Tukey test indicates far " +
                         "outliers in"] = outlier_vars

            # highlight possible multimodal distributions using hartigan's dip
            # test -1 values indicate NaN
            modal_mask = (self.cont_describe.diptest >= 0) & (self.cont_describe.diptest <= 0.05)
            modal_vars = list(self.cont_describe.diptest[modal_mask].dropna(how='all').index)
            if modal_vars:
                warnings["Warning, Hartigan's Dip Test reports possible " +
                         "multimodal distributions for"] = modal_vars

            # highlight non normal distributions
            # -1 values indicate NaN
            modal_mask = (self.cont_describe.normaltest >= 0) & (self.cont_describe.normaltest <= 0.001)
            modal_vars = list(self.cont_describe.normaltest[modal_mask].dropna(how='all').index)
            if modal_vars:
                warnings["Warning, test for normality reports " +
                         "non-normal distributions for"] = modal_vars

        # create the warning string
        for n, k in enumerate(sorted(warnings)):
            msg += '[{}] {}: {}.{}'.format(n+1, k, ', '.join(warnings[k]),
                                           end_of_line)

        return msg 
Example 55
Project: senior-design   Author: james-tate   File: test_stats.py    GNU General Public License v2.0 5 votes vote down vote up
def test_normalitytests():
    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502,  0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    yield assert_array_almost_equal, stats.normaltest(x), (st_normal, pv_normal)
    yield assert_array_almost_equal, stats.skewtest(x), (st_skew, pv_skew)
    yield assert_array_almost_equal, stats.kurtosistest(x), (st_kurt, pv_kurt) 
Example 56
Project: LaserTOF   Author: kyleuckert   File: test_stats.py    MIT License 4 votes vote down vote up
def test_normalitytests():
    yield (assert_raises, ValueError, stats.skewtest, 4.)
    yield (assert_raises, ValueError, stats.kurtosistest, 4.)
    yield (assert_raises, ValueError, stats.normaltest, 4.)

    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502, 0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    attributes = ('statistic', 'pvalue')

    yield assert_array_almost_equal, stats.normaltest(x), (st_normal, pv_normal)
    check_named_results(stats.normaltest(x), attributes)
    yield assert_array_almost_equal, stats.skewtest(x), (st_skew, pv_skew)
    check_named_results(stats.skewtest(x), attributes)
    yield assert_array_almost_equal, stats.kurtosistest(x), (st_kurt, pv_kurt)
    check_named_results(stats.kurtosistest(x), attributes)

    # Test axis=None (equal to axis=0 for 1-D input)
    yield (assert_array_almost_equal, stats.normaltest(x, axis=None),
           (st_normal, pv_normal))
    yield (assert_array_almost_equal, stats.skewtest(x, axis=None),
           (st_skew, pv_skew))
    yield (assert_array_almost_equal, stats.kurtosistest(x, axis=None),
           (st_kurt, pv_kurt))

    x = np.arange(10.)
    x[9] = np.nan
    assert_array_equal(stats.skewtest(x), (np.nan, np.nan))

    expected = (1.0184643553962129, 0.30845733195153502)
    assert_array_almost_equal(stats.skewtest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.skewtest, x, nan_policy='raise')
    assert_raises(ValueError, stats.skewtest, x, nan_policy='foobar')

    x = np.arange(30.)
    x[29] = np.nan
    assert_array_equal(stats.kurtosistest(x), (np.nan, np.nan))

    expected = (-2.2683547379505273, 0.023307594135872967)
    assert_array_almost_equal(stats.kurtosistest(x, nan_policy='omit'),
                              expected)

    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='raise')
    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='foobar')

    assert_array_equal(stats.normaltest(x), (np.nan, np.nan))

    expected = (6.2260409514287449, 0.04446644248650191)
    assert_array_almost_equal(stats.normaltest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.normaltest, x, nan_policy='raise')
    assert_raises(ValueError, stats.normaltest, x, nan_policy='foobar') 
Example 57
Project: poker   Author: surgebiswas   File: test_stats.py    MIT License 4 votes vote down vote up
def test_normalitytests():
    yield (assert_raises, ValueError, stats.skewtest, 4.)
    yield (assert_raises, ValueError, stats.kurtosistest, 4.)
    yield (assert_raises, ValueError, stats.normaltest, 4.)

    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502, 0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    attributes = ('statistic', 'pvalue')

    yield assert_array_almost_equal, stats.normaltest(x), (st_normal, pv_normal)
    check_named_results(stats.normaltest(x), attributes)
    yield assert_array_almost_equal, stats.skewtest(x), (st_skew, pv_skew)
    check_named_results(stats.skewtest(x), attributes)
    yield assert_array_almost_equal, stats.kurtosistest(x), (st_kurt, pv_kurt)
    check_named_results(stats.kurtosistest(x), attributes)

    # Test axis=None (equal to axis=0 for 1-D input)
    yield (assert_array_almost_equal, stats.normaltest(x, axis=None),
           (st_normal, pv_normal))
    yield (assert_array_almost_equal, stats.skewtest(x, axis=None),
           (st_skew, pv_skew))
    yield (assert_array_almost_equal, stats.kurtosistest(x, axis=None),
           (st_kurt, pv_kurt))

    x = np.arange(10.)
    x[9] = np.nan
    assert_array_equal(stats.skewtest(x), (np.nan, np.nan))

    expected = (1.0184643553962129, 0.30845733195153502)
    assert_array_almost_equal(stats.skewtest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.skewtest, x, nan_policy='raise')
    assert_raises(ValueError, stats.skewtest, x, nan_policy='foobar')

    x = np.arange(30.)
    x[29] = np.nan
    assert_array_equal(stats.kurtosistest(x), (np.nan, np.nan))

    expected = (-2.2683547379505273, 0.023307594135872967)
    assert_array_almost_equal(stats.kurtosistest(x, nan_policy='omit'),
                              expected)

    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='raise')
    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='foobar')

    assert_array_equal(stats.normaltest(x), (np.nan, np.nan))

    expected = (6.2260409514287449, 0.04446644248650191)
    assert_array_almost_equal(stats.normaltest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.normaltest, x, nan_policy='raise')
    assert_raises(ValueError, stats.normaltest, x, nan_policy='foobar') 
Example 58
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_stats.py    MIT License 4 votes vote down vote up
def test_normalitytests():
    assert_raises(ValueError, stats.skewtest, 4.)
    assert_raises(ValueError, stats.kurtosistest, 4.)
    assert_raises(ValueError, stats.normaltest, 4.)

    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502, 0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    attributes = ('statistic', 'pvalue')

    assert_array_almost_equal(stats.normaltest(x), (st_normal, pv_normal))
    check_named_results(stats.normaltest(x), attributes)
    assert_array_almost_equal(stats.skewtest(x), (st_skew, pv_skew))
    check_named_results(stats.skewtest(x), attributes)
    assert_array_almost_equal(stats.kurtosistest(x), (st_kurt, pv_kurt))
    check_named_results(stats.kurtosistest(x), attributes)

    # Test axis=None (equal to axis=0 for 1-D input)
    assert_array_almost_equal(stats.normaltest(x, axis=None),
           (st_normal, pv_normal))
    assert_array_almost_equal(stats.skewtest(x, axis=None),
           (st_skew, pv_skew))
    assert_array_almost_equal(stats.kurtosistest(x, axis=None),
           (st_kurt, pv_kurt))

    x = np.arange(10.)
    x[9] = np.nan
    with np.errstate(invalid="ignore"):
        assert_array_equal(stats.skewtest(x), (np.nan, np.nan))

    expected = (1.0184643553962129, 0.30845733195153502)
    assert_array_almost_equal(stats.skewtest(x, nan_policy='omit'), expected)

    with np.errstate(all='ignore'):
        assert_raises(ValueError, stats.skewtest, x, nan_policy='raise')
    assert_raises(ValueError, stats.skewtest, x, nan_policy='foobar')

    x = np.arange(30.)
    x[29] = np.nan
    with np.errstate(all='ignore'):
        assert_array_equal(stats.kurtosistest(x), (np.nan, np.nan))

    expected = (-2.2683547379505273, 0.023307594135872967)
    assert_array_almost_equal(stats.kurtosistest(x, nan_policy='omit'),
                              expected)

    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='raise')
    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='foobar')

    with np.errstate(all='ignore'):
        assert_array_equal(stats.normaltest(x), (np.nan, np.nan))

    expected = (6.2260409514287449, 0.04446644248650191)
    assert_array_almost_equal(stats.normaltest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.normaltest, x, nan_policy='raise')
    assert_raises(ValueError, stats.normaltest, x, nan_policy='foobar') 
Example 59
Project: GSTools   Author: GeoStat-Framework   File: test_rng.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def test_sample_dist(self):
        model = Gaussian(dim=1, var=3.5, len_scale=8.0)
        pdf, cdf, ppf = model.dist_func
        rad = self.rng.sample_dist(
            size=self.few_modes, pdf=pdf, cdf=cdf, ppf=ppf, a=0
        )
        self.assertEqual(rad.shape[0], self.few_modes)

        model = Gaussian(dim=2, var=3.5, len_scale=8.0)
        pdf, cdf, ppf = model.dist_func
        rad = self.rng.sample_dist(
            size=self.few_modes, pdf=pdf, cdf=cdf, ppf=ppf, a=0
        )
        self.assertEqual(rad.shape[0], self.few_modes)

        model = Gaussian(dim=3, var=3.5, len_scale=8.0)
        pdf, cdf, ppf = model.dist_func
        rad = self.rng.sample_dist(
            size=self.few_modes, pdf=pdf, cdf=cdf, ppf=ppf, a=0
        )
        self.assertEqual(rad.shape[0], self.few_modes)

        # model = Gaussian(dim=2, var=3.5, len_scale=8.)
        # pdf, cdf, ppf = model.dist_func
        # rad = self.rng.sample_dist(
        #    size=self.many_modes, pdf=pdf, cdf=cdf, ppf=ppf, a=0)
        # import matplotlib.pyplot as pt
        # pt.hist(rad, bins=30)
        # print(rad)
        # pt.show()

        # TODO test with different models

    # TODO rework this
    # def test_gau(self):
    #    for d in range(len(self.rngs)):
    #        Z, k = self.rngs[d]('gau', self.len_scale, self.many_modes)
    #        self.assertEqual(k.shape, (d+1, self.many_modes))
    #        self.assertAlmostEqual(np.mean(k), 0., places=2)
    #        self.assertAlmostEqual(np.std(k), 1/self.len_scale, places=2)
    #        self.assertAlmostEqual(skew(k[0, :]), 0., places=2)
    #        self.assertAlmostEqual(kurtosis(k[0, :]), 0., places=1)
    #        self.assertLess(normaltest(k[0, :])[1], 0.05) 
Example 60
Project: ChainConsumer   Author: Samreay   File: diagnostic.py    MIT License 4 votes vote down vote up
def geweke(self, chain=None, first=0.1, last=0.5, threshold=0.05):
        """ Runs the Geweke diagnostic on the supplied chains.

        Parameters
        ----------
        chain : int|str, optional
            Which chain to run the diagnostic on. By default, this is `None`,
            which will run the diagnostic on all chains. You can also
            supply and integer (the chain index) or a string, for the chain
            name (if you set one).
        first : float, optional
            The amount of the start of the chain to use
        last : float, optional
            The end amount of the chain to use
        threshold : float, optional
            The p-value to use when testing for normality.

        Returns
        -------
        float
            whether or not the chains pass the test

        """
        if chain is None:
            return np.all([self.geweke(k, threshold=threshold) for k in range(len(self.parent.chains))])

        index = self.parent._get_chain(chain)
        assert len(index) == 1, "Please specify only one chain, have %d chains" % len(index)
        chain = self.parent.chains[index[0]]

        num_walkers = chain.walkers
        assert num_walkers is not None and num_walkers > 0, "You need to specify the number of walkers to use the Geweke diagnostic."
        name = chain.name
        data = chain.chain
        chains = np.split(data, num_walkers)
        n = 1.0 * chains[0].shape[0]
        n_start = int(np.floor(first * n))
        n_end = int(np.floor((1 - last) * n))
        mean_start = np.array([np.mean(c[:n_start, i]) for c in chains for i in range(c.shape[1])])
        var_start = np.array([self._spec(c[:n_start, i]) / c[:n_start, i].size for c in chains for i in range(c.shape[1])])
        mean_end = np.array([np.mean(c[n_end:, i]) for c in chains for i in range(c.shape[1])])
        var_end = np.array([self._spec(c[n_end:, i]) / c[n_end:, i].size for c in chains for i in range(c.shape[1])])
        zs = (mean_start - mean_end) / (np.sqrt(var_start + var_end))
        _, pvalue = normaltest(zs)
        print("Gweke Statistic for chain %s has p-value %e" % (name, pvalue))
        return pvalue > threshold

    # Method of estimating spectral density following PyMC.
    # See https://github.com/pymc-devs/pymc/blob/master/pymc/diagnostics.py 
Example 61
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_stats.py    Apache License 2.0 4 votes vote down vote up
def test_normalitytests():
    yield (assert_raises, ValueError, stats.skewtest, 4.)
    yield (assert_raises, ValueError, stats.kurtosistest, 4.)
    yield (assert_raises, ValueError, stats.normaltest, 4.)

    # numbers verified with R: dagoTest in package fBasics
    st_normal, st_skew, st_kurt = (3.92371918, 1.98078826, -0.01403734)
    pv_normal, pv_skew, pv_kurt = (0.14059673, 0.04761502, 0.98880019)
    x = np.array((-2,-1,0,1,2,3)*4)**2
    attributes = ('statistic', 'pvalue')

    yield assert_array_almost_equal, stats.normaltest(x), (st_normal, pv_normal)
    check_named_results(stats.normaltest(x), attributes)
    yield assert_array_almost_equal, stats.skewtest(x), (st_skew, pv_skew)
    check_named_results(stats.skewtest(x), attributes)
    yield assert_array_almost_equal, stats.kurtosistest(x), (st_kurt, pv_kurt)
    check_named_results(stats.kurtosistest(x), attributes)

    # Test axis=None (equal to axis=0 for 1-D input)
    yield (assert_array_almost_equal, stats.normaltest(x, axis=None),
           (st_normal, pv_normal))
    yield (assert_array_almost_equal, stats.skewtest(x, axis=None),
           (st_skew, pv_skew))
    yield (assert_array_almost_equal, stats.kurtosistest(x, axis=None),
           (st_kurt, pv_kurt))

    x = np.arange(10.)
    x[9] = np.nan
    assert_array_equal(stats.skewtest(x), (np.nan, np.nan))

    expected = (1.0184643553962129, 0.30845733195153502)
    assert_array_almost_equal(stats.skewtest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.skewtest, x, nan_policy='raise')
    assert_raises(ValueError, stats.skewtest, x, nan_policy='foobar')

    x = np.arange(30.)
    x[29] = np.nan
    assert_array_equal(stats.kurtosistest(x), (np.nan, np.nan))

    expected = (-2.2683547379505273, 0.023307594135872967)
    assert_array_almost_equal(stats.kurtosistest(x, nan_policy='omit'),
                              expected)

    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='raise')
    assert_raises(ValueError, stats.kurtosistest, x, nan_policy='foobar')

    assert_array_equal(stats.normaltest(x), (np.nan, np.nan))

    expected = (6.2260409514287449, 0.04446644248650191)
    assert_array_almost_equal(stats.normaltest(x, nan_policy='omit'), expected)

    assert_raises(ValueError, stats.normaltest, x, nan_policy='raise')
    assert_raises(ValueError, stats.normaltest, x, nan_policy='foobar') 
Example 62
Project: AlphaPy   Author: ScottfreeLLC   File: features.py    Apache License 2.0 4 votes vote down vote up
def get_numerical_features(fnum, fname, df, nvalues, dt,
                           sentinel, logt, plevel):
    r"""Transform numerical features with imputation and possibly
    log-transformation.

    Parameters
    ----------
    fnum : int
        Feature number, strictly for logging purposes
    fname : str
        Name of the numerical column in the dataframe ``df``.
    df : pandas.DataFrame
        Dataframe containing the column ``fname``.
    nvalues : int
        The number of unique values.
    dt : str
        The values ``'float64'``, ``'int64'``, or ``'bool'``.
    sentinel : float
        The number to be imputed for NaN values.
    logt : bool
        If ``True``, then log-transform numerical values.
    plevel : float
        The p-value threshold to test if a feature is normally distributed.

    Returns
    -------
    new_values : numpy array
        The set of imputed and transformed features.

    """
    feature = df[fname]
    if len(feature) == nvalues:
        logger.info("Feature %d: %s is a numerical feature of type %s with maximum number of values %d",
                    fnum, fname, dt, nvalues)
    else:
        logger.info("Feature %d: %s is a numerical feature of type %s with %d unique values",
                    fnum, fname, dt, nvalues)
    # imputer for float, integer, or boolean data types
    new_values = impute_values(feature, dt, sentinel)
    # log-transform any values that do not fit a normal distribution
    if logt and np.all(new_values > 0):
        stat, pvalue = sps.normaltest(new_values)
        if pvalue <= plevel:
            logger.info("Feature %d: %s is not normally distributed [p-value: %f]",
                        fnum, fname, pvalue)
            new_values = np.log(new_values)
    return new_values


#
# Function get_polynomials
# 
Example 63
Project: AlphaPy   Author: ScottfreeLLC   File: features.py    Apache License 2.0 4 votes vote down vote up
def create_scipy_features(base_features, sentinel):
    r"""Calculate the skew, kurtosis, and other statistical features
    for each row.

    Parameters
    ----------
    base_features : numpy array
        The feature dataframe.
    sentinel : float
        The number to be imputed for NaN values.

    Returns
    -------
    sp_features : numpy array
        The calculated SciPy features.

    """

    logger.info("Creating SciPy Features")

    # Generate scipy features

    logger.info("SciPy Feature: geometric mean")
    row_gmean = sps.gmean(base_features, axis=1)
    logger.info("SciPy Feature: kurtosis")
    row_kurtosis = sps.kurtosis(base_features, axis=1)
    logger.info("SciPy Feature: kurtosis test")
    row_ktest, pvalue = sps.kurtosistest(base_features, axis=1)
    logger.info("SciPy Feature: normal test")
    row_normal, pvalue = sps.normaltest(base_features, axis=1)
    logger.info("SciPy Feature: skew")
    row_skew = sps.skew(base_features, axis=1)
    logger.info("SciPy Feature: skew test")
    row_stest, pvalue = sps.skewtest(base_features, axis=1)
    logger.info("SciPy Feature: variation")
    row_var = sps.variation(base_features, axis=1)
    logger.info("SciPy Feature: signal-to-noise ratio")
    row_stn = sps.signaltonoise(base_features, axis=1)
    logger.info("SciPy Feature: standard error of mean")
    row_sem = sps.sem(base_features, axis=1)

    sp_features = np.column_stack((row_gmean, row_kurtosis, row_ktest,
                                   row_normal, row_skew, row_stest,
                                   row_var, row_stn, row_sem))
    sp_features = impute_values(sp_features, 'float64', sentinel)
    sp_features = StandardScaler().fit_transform(sp_features)

    # Return new SciPy features

    logger.info("SciPy Feature Count : %d", sp_features.shape[1])
    return sp_features


#
# Function create_clusters
#