Python scipy.stats.ttest_rel() Examples

The following are code examples for showing how to use scipy.stats.ttest_rel(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 2
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 3
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 4
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 5
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 6
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 7
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 6 votes vote down vote up
def test_vs_nonmasked(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        # 1-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1])
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        assert_allclose(res1, res2)

        # 2-D inputs
        res1 = stats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        res2 = mstats.ttest_rel(outcome[:, 0], outcome[:, 1], axis=None)
        assert_allclose(res1, res2)
        res1 = stats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        res2 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:], axis=0)
        assert_allclose(res1, res2)

        # Check default is axis=0
        res3 = mstats.ttest_rel(outcome[:, :2], outcome[:, 2:])
        assert_allclose(res2, res3) 
Example 8
Project: NPRF   Author: ucasir   File: evaluations.py    Apache License 2.0 6 votes vote down vote up
def tt_test(qrels, res1, res2):
  MAP_set1, P20_set1, NDCG20_set1 = evaluate_trec_per_query(qrels, res1)
  MAP_set2, P20_set2, NDCG20_set2 = evaluate_trec_per_query(qrels, res2)
  '''
  print(P20_set1)
  print(P20_set2)
  print(NDCG20_set1)
  print(NDCG20_set2)
  print(len([t for t in np.asarray(MAP_set2) - np.asarray(MAP_set1) if t > 0]))
  print(len([t for t in np.asarray(P20_set2) - np.asarray(P20_set1) if t > 0]))
  print(len([t for t in np.asarray(NDCG20_set2) - np.asarray(NDCG20_set1) if t > 0]))
  '''
  t_value_map, p_value_map = stats.ttest_rel(MAP_set1, MAP_set2)
  t_value_p20, p_value_p20 = stats.ttest_rel(P20_set1, P20_set2)
  t_value_ndcg20, p_value_ndcg20 = stats.ttest_rel(NDCG20_set1, NDCG20_set2)

  return p_value_map, p_value_p20, p_value_ndcg20 
Example 9
Project: learning-paradigms-for-tsp   Author: chaitjo   File: reinforce_baselines.py    MIT License 6 votes vote down vote up
def epoch_callback(self, model, epoch):
        """
        Challenges the current baseline with the model and replaces the baseline model if it is improved.
        :param model: The model to challenge the baseline by
        :param epoch: The current epoch
        """
        print("Evaluating candidate model on evaluation dataset")
        candidate_vals = rollout(model, self.dataset, self.opts).cpu().numpy()

        candidate_mean = candidate_vals.mean()

        print("Epoch {} candidate mean {}, baseline epoch {} mean {}, difference {}".format(
            epoch, candidate_mean, self.epoch, self.mean, candidate_mean - self.mean))
        if candidate_mean - self.mean < 0:
            # Calc p value
            t, p = ttest_rel(candidate_vals, self.bl_vals)

            p_val = p / 2  # one-sided
            assert t < 0, "T-statistic should be negative"
            print("p-value: {}".format(p_val))
            if p_val < self.opts.bl_alpha:
                print('Update baseline')
                self._update_model(model, epoch) 
Example 10
Project: CAVE   Author: automl   File: statistical_tests.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def paired_t_student(data1, data2, logger=None):
    """Test for significance using paired t-test.

    Parameters
    ----------
    data1, data2: List<float>, List<float>
        ordered results for instances for two different configurations
    logger: logging.Logger
        to log scores, if given

    Returns
    -------
    p: float
        p-value for statistical test
    """
    t, p = ttest_rel(data1, data2)
    if logger:
        logger.debug("Paired t-test with %d samples yields t-value of %f and "
                     "p-value of %f", len(data1), t, p)
    return p 
Example 11
Project: InDepth-Analysis   Author: adiyoss   File: check_significance.py    MIT License 6 votes vote down vote up
def calc_significance(path_1, path_2):
    x1 = np.loadtxt(path_1)
    x2 = np.loadtxt(path_2)

    for i in range(len(x1)):
        if x1[i] == 0.0:
            x1[i] = 1.0
        elif x1[i] == 1.0:
            x1[i] = 0.0

        if x2[i] == 0.0:
            x2[i] = 1.0
        elif x2[i] == 1.0:
            x2[i] = 0.0
    t, p = stats.wilcoxon(x1, x2)
    # t, p = stats.ttest_rel(x1, x2)
    # t, p = stats.ttest_ind(x1, x2, equal_var=False)
    return t, p 
Example 12
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore')
            assert_array_equal(mstats.ttest_rel(outcome[:, 0], outcome[:, 1]),
                               (np.nan, np.nan))
            assert_array_equal(mstats.ttest_rel([np.nan, np.nan], [1.0, 2.0]),
                               (np.nan, np.nan)) 
Example 13
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 14
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 15
Project: LaserTOF   Author: kyleuckert   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 16
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 17
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_mstats_basic.py    GNU General Public License v3.0 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 18
Project: NeuralCR   Author: a-arbabi   File: t_test_eval.py    MIT License 5 votes vote down vote up
def paired_test_per_document_fscore(label_dir, output_dir_a, output_dir_b, file_list, ont, column_a, column_b, is_mimic=False, snomed2icd=None):
    dicts = {}
    if not is_mimic:
        dicts['model_a'] = get_matrix_dict(label_dir, output_dir_a, file_list, ont, column=column_a)
        dicts['model_b'] = get_matrix_dict(label_dir, output_dir_b, file_list, ont, column=column_b)
    else:
        dicts['model_a'] = get_matrix_dict_mimic(label_dir, output_dir_a, file_list, ont, snomed2icd, column=column_a)
        dicts['model_b'] = get_matrix_dict_mimic(label_dir, output_dir_b, file_list, ont, snomed2icd, column=column_b)
    heading = [('ord', 'tp'),('ord', 'fp'),('ord', 'rp')]
    f1_list = {'model_a':[],'model_b':[]}
    for filename in sorted(file_list):
        for mod in sorted(dicts.keys()):
            tp = dicts[mod]['ord']['tp'][filename]
            fp = dicts[mod]['ord']['fp'][filename]
            rp = dicts[mod]['ord']['rp'][filename]
            recall = tp/rp if rp!=0 else 1.0
            precision = tp/(tp+fp) if tp+fp!=0 else 1.0
            f1 = get_fmeasure(precision, recall)
            f1_list[mod].append(f1)

    return {'mean_a': np.mean(f1_list['model_a']),
            'mean_b': np.mean(f1_list['model_b']),
            't-test': stats.ttest_rel(
                np.array(f1_list['model_a']), np.array(f1_list['model_b'])),
            'wilcoxon': stats.wilcoxon(
                np.array(f1_list['model_a']), np.array(f1_list['model_b'])),
            'wilcoxon-pratt': stats.wilcoxon(
                np.array(f1_list['model_a']), np.array(f1_list['model_b']),
                zero_method='pratt')} 
Example 19
Project: ACE   Author: anonymousneuripssubmission   File: ace.py    MIT License 5 votes vote down vote up
def do_statistical_testings(self, i_ups_concept, i_ups_random):
    """Conducts ttest to compare two set of samples.

    In particular, if the means of the two samples are staistically different.

    Args:
      i_ups_concept: samples of TCAV scores for concept vs. randoms
      i_ups_random: samples of TCAV scores for random vs. randoms

    Returns:
      p value
    """
    min_len = min(len(i_ups_concept), len(i_ups_random))
    _, p = stats.ttest_rel(i_ups_concept[:min_len], i_ups_random[:min_len])
    return p 
Example 20
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning, "invalid value encountered in absolute")
            for pair in [(outcome[:, 0], outcome[:, 1]), ([np.nan, np.nan], [1.0, 2.0])]:
                t, p = mstats.ttest_rel(*pair)
                assert_array_equal(t, (np.nan, np.nan))
                assert_array_equal(p, (np.nan, np.nan)) 
Example 21
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 22
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 23
Project: ble5-nrf52-mac   Author: tomasero   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 24
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore')
            assert_array_equal(mstats.ttest_rel(outcome[:, 0], outcome[:, 1]),
                               (np.nan, np.nan))
            assert_array_equal(mstats.ttest_rel([np.nan, np.nan], [1.0, 2.0]),
                               (np.nan, np.nan)) 
Example 25
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 26
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 27
Project: poker   Author: surgebiswas   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 28
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning, "invalid value encountered in absolute")
            for pair in [(outcome[:, 0], outcome[:, 1]), ([np.nan, np.nan], [1.0, 2.0])]:
                t, p = mstats.ttest_rel(*pair)
                assert_array_equal(t, (np.nan, np.nan))
                assert_array_equal(p, (np.nan, np.nan)) 
Example 29
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 30
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 31
Project: P3_image_processing   Author: latedude2   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 32
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning, "invalid value encountered in absolute")
            for pair in [(outcome[:, 0], outcome[:, 1]), ([np.nan, np.nan], [1.0, 2.0])]:
                t, p = mstats.ttest_rel(*pair)
                assert_array_equal(t, (np.nan, np.nan))
                assert_array_equal(p, (np.nan, np.nan)) 
Example 33
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 34
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 35
Project: GraphicDesignPatternByPython   Author: Relph1119   File: test_mstats_basic.py    MIT License 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 36
Project: noisyopt   Author: andim   File: main.py    MIT License 5 votes vote down vote up
def test(self, xtest, x, type_='smaller', alpha=0.05):
        """
        Parameters
        ----------
        type_: in ['smaller', 'equality']
            type of comparison to perform
        alpha: float
            significance level
        """
        # call function to make sure it has been evaluated a sufficient number of times
        if type_ not in ['smaller', 'equality']:
            raise NotImplementedError(type_)
        ftest, ftestse = self(xtest)
        f, fse = self(x)
        # get function values
        fxtest = np.array(self.cache[tuple(xtest)])
        fx = np.array(self.cache[tuple(x)])
        if np.mean(fxtest-fx) == 0.0:
            if type_ == 'equality':
                return True
            if type_ == 'smaller':
                return False
        if self.paired:
            # if values are paired then test on distribution of differences
            statistic, pvalue = stats.ttest_rel(fxtest, fx, axis=None)
        else:
            statistic, pvalue = stats.ttest_ind(fxtest, fx, equal_var=False, axis=None)
        if type_ == 'smaller':
            # if paired then df=N-1, else df=N1+N2-2=2*N-2 
            df = self.N-1 if self.paired else 2*self.N-2
            pvalue = stats.t.cdf(statistic, df) 
            # return true if null hypothesis rejected
            return pvalue < alpha
        if type_ == 'equality':
            # return true if null hypothesis not rejected
            return pvalue > alpha 
Example 37
Project: neural-srl   Author: Impavidity   File: eval.py    MIT License 5 votes vote down vote up
def ttest(file_old, file_new):
  fold = open(file_old)
  fnew = open(file_new)
  base = [float(num.strip()) for num in fold.readlines()]
  test = [float(num.strip()) for num in fnew.readlines()]
  if len(base) != len(test):
    print("Length Error")
    exit()
    print("base %d test %d" % (len(base), len(test)))
  a, p = stats.ttest_rel(base, test)
  print("a = %f , p = %f" % (a, p)) 
Example 38
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_fully_masked(self):
        np.random.seed(1234567)
        outcome = ma.masked_array(np.random.randn(3, 2),
                                  mask=[[1, 1, 1], [0, 0, 0]])
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore')
            assert_array_equal(mstats.ttest_rel(outcome[:, 0], outcome[:, 1]),
                               (np.nan, np.nan))
            assert_array_equal(mstats.ttest_rel([np.nan, np.nan], [1.0, 2.0]),
                               (np.nan, np.nan)) 
Example 39
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_result_attributes(self):
        np.random.seed(1234567)
        outcome = np.random.randn(20, 4) + [0, 0, 1, 2]

        res = mstats.ttest_rel(outcome[:, 0], outcome[:, 1])
        attributes = ('statistic', 'pvalue')
        check_named_results(res, attributes, ma=True) 
Example 40
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_invalid_input_size(self):
        assert_raises(ValueError, mstats.ttest_rel,
                      np.arange(10), np.arange(11))
        x = np.arange(24)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=1)
        assert_raises(ValueError, mstats.ttest_rel,
                      x.reshape(2, 3, 4), x.reshape(2, 4, 3), axis=2) 
Example 41
Project: wine-ml-on-aws-lambda   Author: pierreant   File: test_mstats_basic.py    Apache License 2.0 5 votes vote down vote up
def test_empty(self):
        res1 = mstats.ttest_rel([], [])
        assert_(np.all(np.isnan(res1))) 
Example 42
Project: NeuralCR   Author: ccmbioinfo   File: t_test_eval.py    MIT License 5 votes vote down vote up
def paired_test_per_document_fscore(label_dir, output_dir_a, output_dir_b, file_list, ont, column_a, column_b, is_mimic=False, snomed2icd=None):
    dicts = {}
    if not is_mimic:
        dicts['model_a'] = get_matrix_dict(label_dir, output_dir_a, file_list, ont, column=column_a)
        dicts['model_b'] = get_matrix_dict(label_dir, output_dir_b, file_list, ont, column=column_b)
    else:
        dicts['model_a'] = get_matrix_dict_mimic(label_dir, output_dir_a, file_list, ont, snomed2icd, column=column_a)
        dicts['model_b'] = get_matrix_dict_mimic(label_dir, output_dir_b, file_list, ont, snomed2icd, column=column_b)
    heading = [('ord', 'tp'),('ord', 'fp'),('ord', 'rp')]
    f1_list = {'model_a':[],'model_b':[]}
    for filename in sorted(file_list):
        for mod in sorted(dicts.keys()):
            tp = dicts[mod]['ord']['tp'][filename]
            fp = dicts[mod]['ord']['fp'][filename]
            rp = dicts[mod]['ord']['rp'][filename]
            recall = tp/rp if rp!=0 else 1.0
            precision = tp/(tp+fp) if tp+fp!=0 else 1.0
            f1 = get_fmeasure(precision, recall)
            f1_list[mod].append(f1)

    return {'mean_a': np.mean(f1_list['model_a']),
            'mean_b': np.mean(f1_list['model_b']),
            't-test': stats.ttest_rel(
                np.array(f1_list['model_a']), np.array(f1_list['model_b'])),
            'wilcoxon': stats.wilcoxon(
                np.array(f1_list['model_a']), np.array(f1_list['model_b'])),
            'wilcoxon-pratt': stats.wilcoxon(
                np.array(f1_list['model_a']), np.array(f1_list['model_b']),
                zero_method='pratt')} 
Example 43
Project: object_relation_transformer   Author: yahoo   File: report.py    MIT License 5 votes vote down vote up
def _create_run_pair_summary(pair_report_data_list, pair_data_frame):
    # type(List[ReportData], str, DataFrame) -> DataFrame
    run_names = pair_data_frame.columns.levels[0]
    run_summaries = [
        _create_single_run_summary(
            report_data, run_name, pair_data_frame[run_name])
        for (run_name, report_data) in zip(run_names, pair_report_data_list)]
    mean_summaries = [
        pair_data_frame[run_name][ALL_SUMMARY_COLUMNS].mean().to_frame(
            run_name + '.means') for run_name in run_names]
    t_test_results = {column: ttest_rel(
        pair_data_frame[run_names[0]][column],
        pair_data_frame[run_names[1]][column], nan_policy='omit')
        for column in ALL_SUMMARY_COLUMNS}
    sample_size = {column: _count_paired_sample_size(
        pair_data_frame[run_names[0]][column],
        pair_data_frame[run_names[1]][column])
        for column in ALL_SUMMARY_COLUMNS}
    t_statistics = {
        column: statistic_and_p_value[0] for column, statistic_and_p_value in
        t_test_results.items()}
    p_values = {
        column: statistic_and_p_value[1] for column, statistic_and_p_value in
        t_test_results.items()}
    t_statistic_data_frame = Series(t_statistics).to_frame(
        't-test statistic (two-tailed)')
    p_value_data_frame = Series(p_values).to_frame('p-value (two-tailed)')
    sample_size_data_frame = Series(sample_size).to_frame(
        't-test sample size')
    return concat(run_summaries + mean_summaries + [
        t_statistic_data_frame, p_value_data_frame,
        sample_size_data_frame], axis=1) 
Example 44
Project: seqFlow   Author: YaqiangCao   File: callFusion.py    Mozilla Public License 2.0 5 votes vote down vote up
def plotFusion():
    mat = pd.read_table("fusions_RPM.txt", index_col=0)
    celllines = ["9810", "CCLP", "HT", "RBE"]
    lungControl = ["197111L", "197222L", "197333L"]
    lungCancer = ["197111T", "197222T", "197333T"]
    for t in mat.itertuples():
        cs = []
        iccs = []
        ds = {}
        for i, nt in enumerate(t):
            c = mat.columns[i - 1]
            if c in celllines:
                cat = "ICC_cellLine"
            else:
                if c in lungControl:
                    cat = "Lung_Control"
                else:
                    if c in lungCancer:
                        cat = "Lung_Cancer"
                    else:
                        if c.endswith("L"):
                            cat = "ICC_control"
                            cs.append(nt)
                        else:
                            cat = "ICC"
                            iccs.append(nt)
            ds[c] = {"RPM": nt, "cat": cat}
        ds = pd.DataFrame(ds).T
        fig, ax = pylab.subplots()
        p = ttest_rel(cs, iccs)[1]
        sns.stripplot(x="cat", y="RPM", data=ds, jitter=True)
        ax.set_ylim([0, 20])
        ax.set_title("%s\nICC vs control\npaired t-test p-value:%f" %
                     (t[0], p))
        labels = ax.get_xticklabels()
        ax.set_xticklabels(labels, rotation=45)
        print list(labels)
        pylab.savefig("%s.pdf" % t[0]) 
Example 45
Project: csmri-refinement   Author: mseitzer   File: summarize_results.py    Apache License 2.0 5 votes vote down vote up
def statistical_testing(args, metrics_by_input, groups_by_name):
  test_fn = ttest_rel if args.stest_mode == 'ttest' else wilcoxon

  # Get group averages
  samples_by_name = {}
  for name, group in groups_by_name.items():
    gmeans = np.mean([metrics_by_input[inp] for inp in group], axis=0)
    samples_by_name[name] = gmeans

  perms = permutations(samples_by_name.items(), 2)

  if args.sprint:
    print('Performing {}'.format(args.stest_mode))
  tested_names = set()
  pvalues_by_name = {}
  for (n1, s1), (n2, s2) in perms:
    if n1 not in tested_names:
      if args.sprint:
        print('Testing {} against:'.format(n1))
      tested_names.add(n1)

    assert len(s1) == len(s2)

    test = test_fn(s1, s2)
    pvalues_by_name.setdefault(n1, []).append(test.pvalue)
    if args.sprint:
      print('\t{}: {:.4f}'.format(n2, test.pvalue))

  significantly_different_names = []
  for name, pvalues in pvalues_by_name.items():
    if all((p < args.slvl) for p in pvalues):
      significantly_different_names.append(name)
      if args.sprint:
        print(('{} ({:.3f}) has p < {} '
               'for all other inputs').format(name,
                                              samples_by_name[name].mean(),
                                              args.slvl))

  return significantly_different_names 
Example 46
Project: pysystemtrade   Author: robcarver17   File: accounting.py    GNU General Public License v3.0 5 votes vote down vote up
def account_test(ac1, ac2):
    """
    Given two Account like objects performs a two sided t test of normalised returns

    :param ac1: first set of returns
    :type ac1: accountCurve or pd.DataFrame of returns

    :param ac2: second set of returns
    :type ac2: accountCurve or pd.DataFrame of returns

    :returns: 2 tuple: difference in means, t-test results
    """

    common_ts = sorted(set(list(ac1.index)) & set(list(ac2.index)))

    ac1_common = ac1.cumsum().reindex(common_ts, method="ffill").diff().values
    ac2_common = ac2.cumsum().reindex(common_ts, method="ffill").diff().values

    missing_values = [
        idx for idx in range(len(common_ts))
        if (np.isnan(ac1_common[idx]) or np.isnan(ac2_common[idx]))
    ]
    ac1_common = [
        ac1_common[idx] for idx in range(len(common_ts))
        if idx not in missing_values
    ]
    ac2_common = [
        ac2_common[idx] for idx in range(len(common_ts))
        if idx not in missing_values
    ]

    ac1_common = ac1_common / np.nanstd(ac1_common)
    ac2_common = ac2_common / np.nanstd(ac2_common)

    diff = np.mean(ac1_common) - np.mean(ac2_common)

    return (diff, ttest_rel(ac1_common, ac2_common)) 
Example 47
Project: keyphrase-gan   Author: avinsit123   File: t_stat.py    MIT License 5 votes vote down vote up
def main(score_dict_a, score_dict_b, k_list, tag_list):
    for tag in tag_list:
        for k in k_list:
            f1_np_array_a = np.array(score_dict_a['[email protected]{}_{}'.format(k, tag)])
            f1_np_array_b = np.array(score_dict_b['[email protected]{}_{}'.format(k, tag)])
            t_stat, p_value = stats.ttest_rel(f1_np_array_a, f1_np_array_b)
            print("tag: {}, topk: {}, p-value: {}".format(tag, k, p_value)) 
Example 48
Project: att   Author: Centre-Alt-Rendiment-Esportiu   File: test_stats.py    GNU General Public License v3.0 4 votes vote down vote up
def test_ttest_rel():
    # regression test
    tr,pr = 0.81248591389165692, 0.41846234511362157
    tpr = ([tr,-tr],[pr,pr])

    rvs1 = np.linspace(1,100,100)
    rvs2 = np.linspace(1.01,99.989,100)
    rvs1_2D = np.array([np.linspace(1,100,100), np.linspace(1.01,99.989,100)])
    rvs2_2D = np.array([np.linspace(1.01,99.989,100), np.linspace(1,100,100)])

    t,p = stats.ttest_rel(rvs1, rvs2, axis=0)
    assert_array_almost_equal([t,p],(tr,pr))
    t,p = stats.ttest_rel(rvs1_2D.T, rvs2_2D.T, axis=0)
    assert_array_almost_equal([t,p],tpr)
    t,p = stats.ttest_rel(rvs1_2D, rvs2_2D, axis=1)
    assert_array_almost_equal([t,p],tpr)

    # test on 3 dimensions
    rvs1_3D = np.dstack([rvs1_2D,rvs1_2D,rvs1_2D])
    rvs2_3D = np.dstack([rvs2_2D,rvs2_2D,rvs2_2D])
    t,p = stats.ttest_rel(rvs1_3D, rvs2_3D, axis=1)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (2, 3))

    t,p = stats.ttest_rel(np.rollaxis(rvs1_3D,2), np.rollaxis(rvs2_3D,2), axis=2)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (3, 2))

    olderr = np.seterr(all='ignore')
    try:
        # test zero division problem
        t,p = stats.ttest_rel([0,0,0],[1,1,1])
        assert_equal((np.abs(t),p), (np.inf, 0))
        assert_equal(stats.ttest_rel([0,0,0], [0,0,0]), (np.nan, np.nan))

        # check that nan in input array result in nan output
        anan = np.array([[1,np.nan],[-1,1]])
        assert_equal(stats.ttest_ind(anan, np.zeros((2,2))),([0, np.nan], [1,np.nan]))
    finally:
        np.seterr(**olderr)

    # test incorrect input shape raise an error
    x = np.arange(24)
    assert_raises(ValueError, stats.ttest_rel, x.reshape((8, 3)),
                  x.reshape((2, 3, 4))) 
Example 49
Project: avocado-vt   Author: avocado-framework   File: regression.py    GNU General Public License v2.0 4 votes vote down vote up
def getTtestPvalue(self, fs_dict1, fs_dict2, paired=None, ratio=None):
        """
        scipy lib is used to compute p-value of Ttest
        scipy: http://www.scipy.org/
        t-test: http://en.wikipedia.org/wiki/Student's_t-test
        """
        try:
            from scipy import stats
            import numpy as np
        except ImportError:
            print("No python scipy/numpy library installed!")
            return None

        ret = []
        s1 = self._process_files(fs_dict1, self._get_list_self, merge=False)
        s2 = self._process_files(fs_dict2, self._get_list_self, merge=False)
        # s*[line][col] contians items (line*col) of all sample files

        for line in range(len(s1)):
            tmp = []
            if type(s1[line]) != list:
                tmp = s1[line]
            else:
                if len(s1[line][0]) < 2:
                    continue
                for col in range(len(s1[line])):
                    avg1 = self._get_list_avg(s1[line][col])
                    avg2 = self._get_list_avg(s2[line][col])
                    sample1 = np.array(s1[line][col])
                    sample2 = np.array(s2[line][col])
                    warnings.simplefilter("ignore", RuntimeWarning)
                    if (paired):
                        if (ratio):
                            (_, p) = stats.ttest_rel(np.log(sample1), np.log(sample2))
                        else:
                            (_, p) = stats.ttest_rel(sample1, sample2)
                    else:
                        (_, p) = stats.ttest_ind(sample1, sample2)
                    flag = "+"
                    if float(avg1) > float(avg2):
                        flag = "-"
                    tmp.append(flag + "%f" % (1 - p))
                tmp = "|".join(tmp)
            ret.append(tmp)
        return ret 
Example 50
Project: Computable   Author: ktraunmueller   File: test_stats.py    MIT License 4 votes vote down vote up
def test_ttest_rel():
    # regression test
    tr,pr = 0.81248591389165692, 0.41846234511362157
    tpr = ([tr,-tr],[pr,pr])

    rvs1 = np.linspace(1,100,100)
    rvs2 = np.linspace(1.01,99.989,100)
    rvs1_2D = np.array([np.linspace(1,100,100), np.linspace(1.01,99.989,100)])
    rvs2_2D = np.array([np.linspace(1.01,99.989,100), np.linspace(1,100,100)])

    t,p = stats.ttest_rel(rvs1, rvs2, axis=0)
    assert_array_almost_equal([t,p],(tr,pr))
    t,p = stats.ttest_rel(rvs1_2D.T, rvs2_2D.T, axis=0)
    assert_array_almost_equal([t,p],tpr)
    t,p = stats.ttest_rel(rvs1_2D, rvs2_2D, axis=1)
    assert_array_almost_equal([t,p],tpr)

    # test on 3 dimensions
    rvs1_3D = np.dstack([rvs1_2D,rvs1_2D,rvs1_2D])
    rvs2_3D = np.dstack([rvs2_2D,rvs2_2D,rvs2_2D])
    t,p = stats.ttest_rel(rvs1_3D, rvs2_3D, axis=1)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (2, 3))

    t,p = stats.ttest_rel(np.rollaxis(rvs1_3D,2), np.rollaxis(rvs2_3D,2), axis=2)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (3, 2))

    olderr = np.seterr(all='ignore')
    try:
        # test zero division problem
        t,p = stats.ttest_rel([0,0,0],[1,1,1])
        assert_equal((np.abs(t),p), (np.inf, 0))
        assert_equal(stats.ttest_rel([0,0,0], [0,0,0]), (np.nan, np.nan))

        # check that nan in input array result in nan output
        anan = np.array([[1,np.nan],[-1,1]])
        assert_equal(stats.ttest_ind(anan, np.zeros((2,2))),([0, np.nan], [1,np.nan]))
    finally:
        np.seterr(**olderr) 
Example 51
Project: DIVE-backend   Author: MacroConnections   File: numerical_comparison.py    GNU General Public License v3.0 4 votes vote down vote up
def get_valid_tests(equal_var, independent, normal, num_samples):
    '''
    Get valid tests given number of samples and statistical characterization of
    samples:

    Equal variance
    Indepenence
    Normality
    '''
    if num_samples == 1:
        valid_tests = {
            'chisquare': stats.chisquare,
            'power_divergence': stats.power_divergence,
            'kstest': stats.kstest
        }
        if normal:
            valid_tests['input']['one_sample_ttest'] = stats.ttest_1samp

    elif num_samples == 2:
        if independent:
            valid_tests = {
                'mannwhitneyu': stats.mannwhitneyu,
                'kruskal': stats.kruskal,
                'ks_2samp': stats.ks_2samp
            }
            if normal:
                valid_tests['two_sample_ttest'] = stats.ttest_ind
                if equal_var:
                    valid_tests['f_oneway'] = stats.f_oneway
        else:
            valid_tests = {
                'two_sample_ks': stats.ks_2samp,
                'wilcoxon': stats.wilcoxon
            }
            if normal:
                valid_tests['two_sample_related_ttest'] = stats.ttest_rel

    elif num_samples >= 3:
        if independent:
            valid_tests = {
                'kruskal': stats.kruskal
            }
            if normal and equal_var:
                valid_tests['f_oneway'] = stats.f_oneway

        else:
            valid_tests['friedmanchisquare'] = stats.friedmanchisquare

    return valid_tests 
Example 52
Project: conpy   Author: AaltoImagingLanguage   File: stats.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def group_connectivity_ttest(cond1, cond2, df=None, tail=None):
    """Paired t-test comparing connectivity between two conditions.

    Parameters
    ----------
    cond1 : list of VertexConnectivity | LabelConnectivity
        For each subject, the connectivity object corresponding to the first
        experimental condition.
    cond2 : list of VertexConnectivity | LabelConnectivity
        For each subject, the connectivity object corresponding to the second
        experimental condition.
    df : int | None
        Degrees of freedom. If ``None``, defaults to ``n_subjects - 1``.
    tail: 'right' | 'left' | None
        Which tailed t-test to use. If ``None``, two-tailed t-test is
        performed.

    Returns
    -------
    t : ndarray, shape (n_pairs,)
        t-values for all connections.
    pval : ndarray, shape (n_pairs,)
        p-values for all connections.
    """
    if len(cond1) != len(cond2):
        raise ValueError('The number of subjects in each condition must be '
                         'the same.')
    n_subjects = len(cond1)

    # Check compatibility of the connection objects
    pairs1 = cond1[0].pairs
    for con in cond1[1:] + cond2:
        if not np.array_equal(pairs1, con.pairs):
            raise ValueError('Not all Connectivity objects have the same '
                             'connection pairs defined.')

    # Perform a paired t-test
    X1 = np.array([con.data for con in cond1])
    X2 = np.array([con.data for con in cond2])
    t, pval = stats.ttest_rel(X1, X2)

    if not df:
        df = n_subjects - 1

    if tail is not None:
        # Scipy gives out only two-tailed p-values
        if tail == 'right':
            pval = stats.t.cdf(-t, df)
        elif tail == 'left':
            pval = stats.t.cdf(t, df)
        else:
            raise ValueError('Tail must be "right", "left" or None.')
    return t, pval 
Example 53
Project: senior-design   Author: james-tate   File: test_stats.py    GNU General Public License v2.0 4 votes vote down vote up
def test_ttest_rel():
    #regression test
    tr,pr = 0.81248591389165692, 0.41846234511362157
    tpr = ([tr,-tr],[pr,pr])

    rvs1 = np.linspace(1,100,100)
    rvs2 = np.linspace(1.01,99.989,100)
    rvs1_2D = np.array([np.linspace(1,100,100), np.linspace(1.01,99.989,100)])
    rvs2_2D = np.array([np.linspace(1.01,99.989,100), np.linspace(1,100,100)])

    t,p = stats.ttest_rel(rvs1, rvs2, axis=0)
    assert_array_almost_equal([t,p],(tr,pr))
    t,p = stats.ttest_rel(rvs1_2D.T, rvs2_2D.T, axis=0)
    assert_array_almost_equal([t,p],tpr)
    t,p = stats.ttest_rel(rvs1_2D, rvs2_2D, axis=1)
    assert_array_almost_equal([t,p],tpr)

    #test on 3 dimensions
    rvs1_3D = np.dstack([rvs1_2D,rvs1_2D,rvs1_2D])
    rvs2_3D = np.dstack([rvs2_2D,rvs2_2D,rvs2_2D])
    t,p = stats.ttest_rel(rvs1_3D, rvs2_3D, axis=1)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (2, 3))

    t,p = stats.ttest_rel(np.rollaxis(rvs1_3D,2), np.rollaxis(rvs2_3D,2), axis=2)
    assert_array_almost_equal(np.abs(t), tr)
    assert_array_almost_equal(np.abs(p), pr)
    assert_equal(t.shape, (3, 2))

    olderr = np.seterr(all='ignore')
    try:
        #test zero division problem
        t,p = stats.ttest_rel([0,0,0],[1,1,1])
        assert_equal((np.abs(t),p), (np.inf, 0))
        assert_almost_equal(stats.ttest_rel([0,0,0], [0,0,0]), (1.0, 0.42264973081037421))

        #check that nan in input array result in nan output
        anan = np.array([[1,np.nan],[-1,1]])
        assert_equal(stats.ttest_ind(anan, np.zeros((2,2))),([0, np.nan], [1,np.nan]))
    finally:
        np.seterr(**olderr)