Python scipy.stats.ttest_rel() Examples

def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 

def paired_t_student(data1, data2, logger=None):
    """Test for significance using paired t-test.

    Parameters
    ----------
    data1, data2: List<float>, List<float>
        ordered results for instances for two different configurations
    logger: logging.Logger
        to log scores, if given

    Returns
    -------
    p: float
        p-value for statistical test
    """
    t, p = ttest_rel(data1, data2)
    if logger:
        logger.debug("Paired t-test with %d samples yields t-value of %f and "
                     "p-value of %f", len(data1), t, p)
    return p 

def tt_test(qrels, res1, res2):
  MAP_set1, P20_set1, NDCG20_set1 = evaluate_trec_per_query(qrels, res1)
  MAP_set2, P20_set2, NDCG20_set2 = evaluate_trec_per_query(qrels, res2)
  '''
  print(P20_set1)
  print(P20_set2)
  print(NDCG20_set1)
  print(NDCG20_set2)
  print(len([t for t in np.asarray(MAP_set2) - np.asarray(MAP_set1) if t > 0]))
  print(len([t for t in np.asarray(P20_set2) - np.asarray(P20_set1) if t > 0]))
  print(len([t for t in np.asarray(NDCG20_set2) - np.asarray(NDCG20_set1) if t > 0]))
  '''
  t_value_map, p_value_map = stats.ttest_rel(MAP_set1, MAP_set2)
  t_value_p20, p_value_p20 = stats.ttest_rel(P20_set1, P20_set2)
  t_value_ndcg20, p_value_ndcg20 = stats.ttest_rel(NDCG20_set1, NDCG20_set2)

  return p_value_map, p_value_p20, p_value_ndcg20 

def ttest_alt(a, b, alternative='two-sided'):
    tt, tp = ttest_rel(a, b)

    if alternative == 'greater':
        if tt > 0:
            tp = 1 - (1-tp)/2
        else:
            tp /= 2
    elif alternative == 'less':
        if tt <= 0:
            tp /= 2
        else:
            tp = 1 - (1-tp)/2

    return tt, tp

################################################################################
# __main__
################################################################################ 

def test(self, xtest, x, type_='smaller', alpha=0.05):
        """
        Parameters
        ----------
        type_: in ['smaller', 'equality']
            type of comparison to perform
        alpha: float
            significance level
        """
        # call function to make sure it has been evaluated a sufficient number of times
        if type_ not in ['smaller', 'equality']:
            raise NotImplementedError(type_)
        ftest, ftestse = self(xtest)
        f, fse = self(x)
        # get function values
        fxtest = np.array(self.cache[tuple(xtest)])
        fx = np.array(self.cache[tuple(x)])
        if np.mean(fxtest-fx) == 0.0:
            if type_ == 'equality':
                return True
            if type_ == 'smaller':
                return False
        if self.paired:
            # if values are paired then test on distribution of differences
            statistic, pvalue = stats.ttest_rel(fxtest, fx, axis=None)
        else:
            statistic, pvalue = stats.ttest_ind(fxtest, fx, equal_var=False, axis=None)
        if type_ == 'smaller':
            # if paired then df=N-1, else df=N1+N2-2=2*N-2 
            df = self.N-1 if self.paired else 2*self.N-2
            pvalue = stats.t.cdf(statistic, df) 
            # return true if null hypothesis rejected
            return pvalue < alpha
        if type_ == 'equality':
            # return true if null hypothesis not rejected
            return pvalue > alpha 

def permttest(data1,data2,permutations, teststat):
    rs=[]
    wtf1=data1
    wtf2=data2
    for perm in range(0,permutations):
        temp1=[]
        temp2=[]
        buf1=wtf1
        buf2=wtf2
        while len(temp1) < len(buf1):
            temp1.append(choice(buf1+buf2))
        #temp1.append(buf1[0])

        while len(temp2) < len(buf2):
            temp2.append(choice(buf2+buf1))
        #temp2.append(buf2[0])
        tval,pval=stats.ttest_rel(temp1,temp2)
        rs.append(tval)
        #print("Computed permutation "+str(perm)+" of "+str(permutations))
    pval=float(0)

    for item in rs:
        if (abs(item) >= abs(teststat)):
            pval=pval+1
    pval=float(pval)/float(permutations)
    return pval 

def do_statistical_testings(self, i_ups_concept, i_ups_random):
    """Conducts ttest to compare two set of samples.

    In particular, if the means of the two samples are staistically different.

    Args:
      i_ups_concept: samples of TCAV scores for concept vs. randoms
      i_ups_random: samples of TCAV scores for random vs. randoms

    Returns:
      p value
    """
    min_len = min(len(i_ups_concept), len(i_ups_random))
    _, p = stats.ttest_rel(i_ups_concept[:min_len], i_ups_random[:min_len])
    return p 

def main(score_dict_a, score_dict_b, k_list, tag_list):
    for tag in tag_list:
        for k in k_list:
            f1_np_array_a = np.array(score_dict_a['f1_score@{}_{}'.format(k, tag)])
            f1_np_array_b = np.array(score_dict_b['f1_score@{}_{}'.format(k, tag)])
            t_stat, p_value = stats.ttest_rel(f1_np_array_a, f1_np_array_b)
            print("tag: {}, topk: {}, p-value: {}".format(tag, k, p_value)) 

def main(score_dict_a, score_dict_b, k_list, tag_list):
    for tag in tag_list:
        for k in k_list:
            f1_np_array_a = np.array(score_dict_a['f1_score@{}_{}'.format(k, tag)])
            f1_np_array_b = np.array(score_dict_b['f1_score@{}_{}'.format(k, tag)])
            t_stat, p_value = stats.ttest_rel(f1_np_array_a, f1_np_array_b)
            print("tag: {}, topk: {}, p-value: {}".format(tag, k, p_value)) 

def account_test(ac1, ac2):
    """
    Given two Account like objects performs a two sided t test of normalised returns

    :param ac1: first set of returns
    :type ac1: accountCurve or pd.DataFrame of returns

    :param ac2: second set of returns
    :type ac2: accountCurve or pd.DataFrame of returns

    :returns: 2 tuple: difference in means, t-test results
    """

    common_ts = sorted(set(list(ac1.index)) & set(list(ac2.index)))

    ac1_common = ac1.cumsum().reindex(common_ts, method="ffill").diff().values
    ac2_common = ac2.cumsum().reindex(common_ts, method="ffill").diff().values

    missing_values = [
        idx for idx in range(len(common_ts))
        if (np.isnan(ac1_common[idx]) or np.isnan(ac2_common[idx]))
    ]
    ac1_common = [
        ac1_common[idx] for idx in range(len(common_ts))
        if idx not in missing_values
    ]
    ac2_common = [
        ac2_common[idx] for idx in range(len(common_ts))
        if idx not in missing_values
    ]

    ac1_common = ac1_common / np.nanstd(ac1_common)
    ac2_common = ac2_common / np.nanstd(ac2_common)

    diff = np.mean(ac1_common) - np.mean(ac2_common)

    return (diff, ttest_rel(ac1_common, ac2_common)) 

def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 

def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 

def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 

def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning, "invalid value encountered in absolute")
            for pair in [(outcome[:, 0], outcome[:, 1]), ([np.nan, np.nan], [1.0, 2.0])]:
                t, p = mstats.ttest_rel(*pair)
                assert_array_equal(t, (np.nan, np.nan))
                assert_array_equal(p, (np.nan, np.nan)) 

def group_connectivity_ttest(cond1, cond2, df=None, tail=None):
    """Paired t-test comparing connectivity between two conditions.

    Parameters
    ----------
    cond1 : list of VertexConnectivity | LabelConnectivity
        For each subject, the connectivity object corresponding to the first
        experimental condition.
    cond2 : list of VertexConnectivity | LabelConnectivity
        For each subject, the connectivity object corresponding to the second
        experimental condition.
    df : int | None
        Degrees of freedom. If ``None``, defaults to ``n_subjects - 1``.
    tail: 'right' | 'left' | None
        Which tailed t-test to use. If ``None``, two-tailed t-test is
        performed.

    Returns
    -------
    t : ndarray, shape (n_pairs,)
        t-values for all connections.
    pval : ndarray, shape (n_pairs,)
        p-values for all connections.
    """
    if len(cond1) != len(cond2):
        raise ValueError('The number of subjects in each condition must be '
                         'the same.')
    n_subjects = len(cond1)

    # Check compatibility of the connection objects
    pairs1 = cond1[0].pairs
    for con in cond1[1:] + cond2:
        if not np.array_equal(pairs1, con.pairs):
            raise ValueError('Not all Connectivity objects have the same '
                             'connection pairs defined.')

    # Perform a paired t-test
    X1 = np.array([con.data for con in cond1])
    X2 = np.array([con.data for con in cond2])
    t, pval = stats.ttest_rel(X1, X2)

    if not df:
        df = n_subjects - 1

    if tail is not None:
        # Scipy gives out only two-tailed p-values
        if tail == 'right':
            pval = stats.t.cdf(-t, df)
        elif tail == 'left':
            pval = stats.t.cdf(t, df)
        else:
            raise ValueError('Tail must be "right", "left" or None.')
    return t, pval 

def get_valid_tests(equal_var, independent, normal, num_samples):
    '''
    Get valid tests given number of samples and statistical characterization of
    samples:

    Equal variance
    Indepenence
    Normality
    '''
    if num_samples == 1:
        valid_tests = {
            'chisquare': stats.chisquare,
            'power_divergence': stats.power_divergence,
            'kstest': stats.kstest
        }
        if normal:
            valid_tests['input']['one_sample_ttest'] = stats.ttest_1samp

    elif num_samples == 2:
        if independent:
            valid_tests = {
                'mannwhitneyu': stats.mannwhitneyu,
                'kruskal': stats.kruskal,
                'ks_2samp': stats.ks_2samp
            }
            if normal:
                valid_tests['two_sample_ttest'] = stats.ttest_ind
                if equal_var:
                    valid_tests['f_oneway'] = stats.f_oneway
        else:
            valid_tests = {
                'two_sample_ks': stats.ks_2samp,
                'wilcoxon': stats.wilcoxon
            }
            if normal:
                valid_tests['two_sample_related_ttest'] = stats.ttest_rel

    elif num_samples >= 3:
        if independent:
            valid_tests = {
                'kruskal': stats.kruskal
            }
            if normal and equal_var:
                valid_tests['f_oneway'] = stats.f_oneway

        else:
            valid_tests['friedmanchisquare'] = stats.friedmanchisquare

    return valid_tests 

def test_ttest_rel():
    # regression test
    tr,pr = 0.81248591389165692, 0.41846234511362157
    tpr = ([tr,-tr],[pr,pr])

    rvs1 = np.linspace(1,100,100)
    rvs2 = np.linspace(1.01,99.989,100)
    rvs1_2D = np.array([np.linspace(1,100,100), np.linspace(1.01,99.989,100)])
    rvs2_2D = np.array([np.linspace(1.01,99.989,100), np.linspace(1,100,100)])

    t,p = stats.ttest_rel(rvs1, rvs2, axis=0)
    assert_array_almost_equal([t,p],(tr,pr))
    t,p = stats.ttest_rel(rvs1_2D.T, rvs2_2D.T, axis=0)
    assert_array_almost_equal([t,p],tpr)
    t,p = stats.ttest_rel(rvs1_2D, rvs2_2D, axis=1)
    assert_array_almost_equal([t,p],tpr)

    # test on 3 dimensions
    rvs1_3D = np.dstack([rvs1_2D,rvs1_2D,rvs1_2D])
    rvs2_3D = np.dstack([rvs2_2D,rvs2_2D,rvs2_2D])
    t,p = stats.ttest_rel(rvs1_3D, rvs2_3D, axis=1)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (2, 3))

    t,p = stats.ttest_rel(np.rollaxis(rvs1_3D,2), np.rollaxis(rvs2_3D,2), axis=2)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (3, 2))

    olderr = np.seterr(all='ignore')
    try:
        # test zero division problem
        t,p = stats.ttest_rel([0,0,0],[1,1,1])
        assert_equal((np.abs(t),p), (np.inf, 0))
        assert_equal(stats.ttest_rel([0,0,0], [0,0,0]), (np.nan, np.nan))

        # check that nan in input array result in nan output
        anan = np.array([[1,np.nan],[-1,1]])
        assert_equal(stats.ttest_ind(anan, np.zeros((2,2))),([0, np.nan], [1,np.nan]))
    finally:
        np.seterr(**olderr) 

def getTtestPvalue(self, fs_dict1, fs_dict2, paired=None, ratio=None):
        """
        scipy lib is used to compute p-value of Ttest
        scipy: http://www.scipy.org/
        t-test: http://en.wikipedia.org/wiki/Student's_t-test
        """
        try:
            from scipy import stats
            import numpy as np
        except ImportError:
            print("No python scipy/numpy library installed!")
            return None

        ret = []
        s1 = self._process_files(fs_dict1, self._get_list_self, merge=False)
        s2 = self._process_files(fs_dict2, self._get_list_self, merge=False)
        # s*[line][col] contians items (line*col) of all sample files

        for line in range(len(s1)):
            tmp = []
            if type(s1[line]) != list:
                tmp = s1[line]
            else:
                if len(s1[line][0]) < 2:
                    continue
                for col in range(len(s1[line])):
                    avg1 = self._get_list_avg(s1[line][col])
                    avg2 = self._get_list_avg(s2[line][col])
                    sample1 = np.array(s1[line][col])
                    sample2 = np.array(s2[line][col])
                    warnings.simplefilter("ignore", RuntimeWarning)
                    if (paired):
                        if (ratio):
                            (_, p) = stats.ttest_rel(np.log(sample1), np.log(sample2))
                        else:
                            (_, p) = stats.ttest_rel(sample1, sample2)
                    else:
                        (_, p) = stats.ttest_ind(sample1, sample2)
                    flag = "+"
                    if float(avg1) > float(avg2):
                        flag = "-"
                    tmp.append(flag + "%f" % (1 - p))
                tmp = "|".join(tmp)
            ret.append(tmp)
        return ret