Python numpy.nanmedian() Examples

def test_equ_med_far_vector(self):
        """
        Test that median reddening is correct in the far limit, using a vector
        of coordinates as input.
        """
        l = [d['l']*units.deg for d in self._test_data]
        b = [d['b']*units.deg for d in self._test_data]
        dist = [1.e3*units.kpc for bb in b]
        c = coords.SkyCoord(l, b, distance=dist, frame='galactic')

        ebv_data = np.array([np.nanmedian(d['samples'][:,-1]) for d in self._test_data])
        ebv_calc = self._bayestar(c, mode='median')

        # print 'vector:'
        # print r'% residual:'
        # for ed,ec in zip(ebv_data, ebv_calc):
        #     print '  {: >8.3f}'.format((ec - ed) / (0.02 + 0.02 * ed))

        np.testing.assert_allclose(ebv_data, ebv_calc, atol=0.001, rtol=0.0001) 

def test_equ_med_scalar(self):
        """
        Test that median reddening is correct in at arbitary distances, using
        individual coordinates as input.
        """
        for d in self._test_data:
            l = d['l']*units.deg
            b = d['b']*units.deg

            for reps in range(10):
                dm = 3. + (25.-3.)*np.random.random()
                dist = 10.**(dm/5.-2.)
                c = coords.SkyCoord(l, b, distance=dist*units.kpc, frame='galactic')

                ebv_samples = self._interp_ebv(d, dist)
                ebv_data = np.nanmedian(ebv_samples)
                ebv_calc = self._bayestar(c, mode='median')

                np.testing.assert_allclose(ebv_data, ebv_calc, atol=0.001, rtol=0.0001) 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        # benchmarked with shuffled (50, 50, x) containing a few NaN
        if a.shape[axis] < 600:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def test_allnans(self):
        mat = np.array([np.nan]*9).reshape(3, 3)
        for axis in [None, 0, 1]:
            with suppress_warnings() as sup:
                sup.record(RuntimeWarning)

                assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
                if axis is None:
                    assert_(len(sup.log) == 1)
                else:
                    assert_(len(sup.log) == 3)
                # Check scalar
                assert_(np.isnan(np.nanmedian(np.nan)))
                if axis is None:
                    assert_(len(sup.log) == 2)
                else:
                    assert_(len(sup.log) == 4) 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        # benchmarked with shuffled (50, 50, x) containing a few NaN
        if a.shape[axis] < 600:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def _nanmedian1d(arr1d, overwrite_input=False):
    """
    Private function for rank 1 arrays. Compute the median ignoring NaNs.
    See nanmedian for parameter usage
    """
    c = np.isnan(arr1d)
    s = np.where(c)[0]
    if s.size == arr1d.size:
        warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=3)
        return np.nan
    elif s.size == 0:
        return np.median(arr1d, overwrite_input=overwrite_input)
    else:
        if overwrite_input:
            x = arr1d
        else:
            x = arr1d.copy()
        # select non-nans at end of array
        enonan = arr1d[-s.size:][~c[-s.size:]]
        # fill nans in beginning of array with non-nans of end
        x[s[:enonan.size]] = enonan
        # slice nans away
        return np.median(x[:-s.size], overwrite_input=True) 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        # benchmarked with shuffled (50, 50, x) containing a few NaN
        if a.shape[axis] < 600:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def test_out(self):
        mat = np.random.rand(3, 3)
        nan_mat = np.insert(mat, [0, 2], np.nan, axis=1)
        resout = np.zeros(3)
        tgt = np.median(mat, axis=1)
        res = np.nanmedian(nan_mat, axis=1, out=resout)
        assert_almost_equal(res, resout)
        assert_almost_equal(res, tgt)
        # 0-d output:
        resout = np.zeros(())
        tgt = np.median(mat, axis=None)
        res = np.nanmedian(nan_mat, axis=None, out=resout)
        assert_almost_equal(res, resout)
        assert_almost_equal(res, tgt)
        res = np.nanmedian(nan_mat, axis=(0, 1), out=resout)
        assert_almost_equal(res, resout)
        assert_almost_equal(res, tgt) 

def _nanmedian1d(arr1d, overwrite_input=False):
    """
    Private function for rank 1 arrays. Compute the median ignoring NaNs.
    See nanmedian for parameter usage
    """
    c = np.isnan(arr1d)
    s = np.where(c)[0]
    if s.size == arr1d.size:
        warnings.warn("All-NaN slice encountered", RuntimeWarning)
        return np.nan
    elif s.size == 0:
        return np.median(arr1d, overwrite_input=overwrite_input)
    else:
        if overwrite_input:
            x = arr1d
        else:
            x = arr1d.copy()
        # select non-nans at end of array
        enonan = arr1d[-s.size:][~c[-s.size:]]
        # fill nans in beginning of array with non-nans of end
        x[s[:enonan.size]] = enonan
        # slice nans away
        return np.median(x[:-s.size], overwrite_input=True) 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        if a.shape[axis] < 400:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def test_allnans(self):
        mat = np.array([np.nan]*9).reshape(3, 3)
        for axis in [None, 0, 1]:
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter('always')
                warnings.simplefilter('ignore', FutureWarning)
                assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
                if axis is None:
                    assert_(len(w) == 1)
                else:
                    assert_(len(w) == 3)
                assert_(issubclass(w[0].category, RuntimeWarning))
                # Check scalar
                assert_(np.isnan(np.nanmedian(np.nan)))
                if axis is None:
                    assert_(len(w) == 2)
                else:
                    assert_(len(w) == 4)
                assert_(issubclass(w[0].category, RuntimeWarning)) 

def compute_mean_metrics(json_folder, compute_averages=True):
    files = glob.glob(os.path.join(json_folder, "*.json"))
    sdr_inst_list = None
    for path in files:
        #print(path)
        with open(path, "r") as f:
            js = json.load(f)

        if sdr_inst_list is None:
            sdr_inst_list = [list() for _ in range(len(js["targets"]))]

        for i in range(len(js["targets"])):
            sdr_inst_list[i].extend([np.float(f['metrics']["SDR"]) for f in js["targets"][i]["frames"]])

    #return np.array(sdr_acc), np.array(sdr_voc)
    sdr_inst_list = [np.array(sdr) for sdr in sdr_inst_list]

    if compute_averages:
        return [(np.nanmedian(sdr), np.nanmedian(np.abs(sdr - np.nanmedian(sdr))), np.nanmean(sdr), np.nanstd(sdr)) for sdr in sdr_inst_list]
    else:
        return sdr_inst_list 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        # benchmarked with shuffled (50, 50, x) containing a few NaN
        if a.shape[axis] < 600:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def test_allnans(self):
        mat = np.array([np.nan]*9).reshape(3, 3)
        for axis in [None, 0, 1]:
            with suppress_warnings() as sup:
                sup.record(RuntimeWarning)

                assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
                if axis is None:
                    assert_(len(sup.log) == 1)
                else:
                    assert_(len(sup.log) == 3)
                # Check scalar
                assert_(np.isnan(np.nanmedian(np.nan)))
                if axis is None:
                    assert_(len(sup.log) == 2)
                else:
                    assert_(len(sup.log) == 4) 

def _nanmedian(a, axis=None, out=None, overwrite_input=False):
    """
    Private function that doesn't support extended axis or keepdims.
    These methods are extended to this function using _ureduce
    See nanmedian for parameter usage

    """
    if axis is None or a.ndim == 1:
        part = a.ravel()
        if out is None:
            return _nanmedian1d(part, overwrite_input)
        else:
            out[...] = _nanmedian1d(part, overwrite_input)
            return out
    else:
        # for small medians use sort + indexing which is still faster than
        # apply_along_axis
        # benchmarked with shuffled (50, 50, x) containing a few NaN
        if a.shape[axis] < 600:
            return _nanmedian_small(a, axis, out, overwrite_input)
        result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)
        if out is not None:
            out[...] = result
        return result 

def test_allnans(self):
        mat = np.array([np.nan]*9).reshape(3, 3)
        for axis in [None, 0, 1]:
            with suppress_warnings() as sup:
                sup.record(RuntimeWarning)

                assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
                if axis is None:
                    assert_(len(sup.log) == 1)
                else:
                    assert_(len(sup.log) == 3)
                # Check scalar
                assert_(np.isnan(np.nanmedian(np.nan)))
                if axis is None:
                    assert_(len(sup.log) == 2)
                else:
                    assert_(len(sup.log) == 4) 

def test_equ_med_far_scalar(self):
        """
        Test that median reddening is correct in the far limit, using a single
        location on the sky at a time as input.
        """
        for d in self._test_data:
            c = self._get_gal(d, dist=1.e3*units.kpc)
            # print d['samples']
            ebv_data = np.nanmedian(d['samples'][:,-1])
            ebv_calc = self._bayestar(c, mode='median')
            # print 'ebv_data:', ebv_data
            # print 'ebv_calc:', ebv_calc
            # print ''
            # print r'% residual: {:.6f}'.format((ebv_calc - ebv_data) / (0.001 + 0.001 * ebv_data))
            np.testing.assert_allclose(ebv_data, ebv_calc, atol=0.001, rtol=0.0001) 

def test_equ_med_vector(self):
        """
        Test that median reddening is correct at arbitary distances, using a
        vector of coordinates as input.
        """
        for reps in range(10):
            l = [d['l']*units.deg for d in self._test_data]
            b = [d['b']*units.deg for d in self._test_data]
            dm = 3. + (25.-3.)*np.random.random(len(self._test_data))
            dist = [d*units.kpc for d in 10.**(dm/5.-2.)]
            dist_unitless = [d for d in 10.**(dm/5.-2.)]
            c = coords.SkyCoord(l, b, distance=dist, frame='galactic')

            ebv_samples = np.array([
                self._interp_ebv(datum, d)
                for datum,d in zip(self._test_data, dist_unitless)
            ])
            ebv_data = np.nanmedian(ebv_samples, axis=1)
            ebv_calc = self._bayestar(c, mode='median')

            # print 'vector arbitrary distance:'
            # print r'% residual:'
            # for ed,ec in zip(ebv_data, ebv_calc):
            #     print '  {: >8.3f}'.format((ec - ed) / (0.02 + 0.02 * ed))

            np.testing.assert_allclose(ebv_data, ebv_calc, atol=0.001, rtol=0.0001) 

def _rsp_rrv_time(bbi):
    diff_bbi = np.diff(bbi)
    out = {}  # Initialize empty dict

    # Mean based
    out["RMSSD"] = np.sqrt(np.mean(diff_bbi ** 2))

    out["MeanBB"] = np.nanmean(bbi)
    out["SDBB"] = np.nanstd(bbi, ddof=1)
    out["SDSD"] = np.nanstd(diff_bbi, ddof=1)

    out["CVBB"] = out["SDBB"] / out["MeanBB"]
    out["CVSD"] = out["RMSSD"] / out["MeanBB"]

    # Robust
    out["MedianBB"] = np.nanmedian(bbi)
    out["MadBB"] = mad(bbi)
    out["MCVBB"] = out["MadBB"] / out["MedianBB"]

    #    # Extreme-based
    #    nn50 = np.sum(np.abs(diff_rri) > 50)
    #    nn20 = np.sum(np.abs(diff_rri) > 20)
    #    out["pNN50"] = nn50 / len(rri) * 100
    #    out["pNN20"] = nn20 / len(rri) * 100
    #
    #    # Geometrical domain
    #    bar_y, bar_x = np.histogram(rri, bins=range(300, 2000, 8))
    #    bar_y, bar_x = np.histogram(rri, bins="auto")
    #    out["TINN"] = np.max(bar_x) - np.min(bar_x)  # Triangular Interpolation of the NN Interval Histogram
    #    out["HTI"] = len(rri) / np.max(bar_y)  # HRV Triangular Index

    return out 

def mad(x, constant=1.4826):
    """Median Absolute Deviation: a "robust" version of standard deviation.

    Parameters
    ----------
    x : Union[list, np.array, pd.Series]
        A vector of values.
    constant : float
        Scale factor. Use 1.4826 for results similar to default R.

    Returns
    ----------
    float
        The MAD.

    Examples
    ----------
    >>> import neurokit2 as nk
    >>> nk.mad([2, 8, 7, 5, 4, 12, 5, 1])
    3.7064999999999997

    References
    -----------
    - https://en.wikipedia.org/wiki/Median_absolute_deviation

    """
    median = np.nanmedian(np.ma.array(x).compressed())
    mad_value = np.nanmedian(np.abs(x - median))
    mad_value = mad_value * constant
    return mad_value 

def _standardize(data, robust=False, window=None, **kwargs):

    # Compute standardized on whole data
    if window is None:
        if robust is False:
            z = (data - np.nanmean(data, axis=0)) / np.nanstd(data, axis=0, ddof=1)
        else:
            z = (data - np.nanmedian(data, axis=0)) / mad(data)

    # Rolling standardization on windows
    else:
        df = pd.DataFrame(data)  # Force dataframe

        if robust is False:
            z = (df - df.rolling(window, min_periods=0, **kwargs).mean()) / df.rolling(
                window, min_periods=0, **kwargs
            ).std(ddof=1)
        else:
            z = (df - df.rolling(window, min_periods=0, **kwargs).median()) / df.rolling(
                window, min_periods=0, **kwargs
            ).apply(mad)

        # Fill the created nans
        z = z.fillna(method="bfill")

        # Restore to vector or array
        if z.shape[1] == 1:
            z = z[0].values
        else:
            z = z.values

    return z 

def _nanmedian1d(arr1d, overwrite_input=False):
    """
    Private function for rank 1 arrays. Compute the median ignoring NaNs.
    See nanmedian for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.nan

    return np.median(arr1d, overwrite_input=overwrite_input) 

def _nanmedian_small(a, axis=None, out=None, overwrite_input=False):
    """
    sort + indexing median, faster for small medians along multiple
    dimensions due to the high overhead of apply_along_axis

    see nanmedian for parameter usage
    """
    a = np.ma.masked_array(a, np.isnan(a))
    m = np.ma.median(a, axis=axis, overwrite_input=overwrite_input)
    for i in range(np.count_nonzero(m.mask.ravel())):
        warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=3)
    if out is not None:
        out[...] = m.filled(np.nan)
        return out
    return m.filled(np.nan) 

def test_mutation(self):
        # Check that passed array is not modified.
        ndat = _ndat.copy()
        np.nanmedian(ndat)
        assert_equal(ndat, _ndat) 

def test_keepdims(self):
        mat = np.eye(3)
        for axis in [None, 0, 1]:
            tgt = np.median(mat, axis=axis, out=None, overwrite_input=False)
            res = np.nanmedian(mat, axis=axis, out=None, overwrite_input=False)
            assert_(res.ndim == tgt.ndim)

        d = np.ones((3, 5, 7, 11))
        # Randomly set some elements to NaN:
        w = np.random.random((4, 200)) * np.array(d.shape)[:, None]
        w = w.astype(np.intp)
        d[tuple(w)] = np.nan
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning)
            res = np.nanmedian(d, axis=None, keepdims=True)
            assert_equal(res.shape, (1, 1, 1, 1))
            res = np.nanmedian(d, axis=(0, 1), keepdims=True)
            assert_equal(res.shape, (1, 1, 7, 11))
            res = np.nanmedian(d, axis=(0, 3), keepdims=True)
            assert_equal(res.shape, (1, 5, 7, 1))
            res = np.nanmedian(d, axis=(1,), keepdims=True)
            assert_equal(res.shape, (3, 1, 7, 11))
            res = np.nanmedian(d, axis=(0, 1, 2, 3), keepdims=True)
            assert_equal(res.shape, (1, 1, 1, 1))
            res = np.nanmedian(d, axis=(0, 1, 3), keepdims=True)
            assert_equal(res.shape, (1, 1, 7, 1)) 

def test_small_large(self):
        # test the small and large code paths, current cutoff 400 elements
        for s in [5, 20, 51, 200, 1000]:
            d = np.random.randn(4, s)
            # Randomly set some elements to NaN:
            w = np.random.randint(0, d.size, size=d.size // 5)
            d.ravel()[w] = np.nan
            d[:,0] = 1.  # ensure at least one good value
            # use normal median without nans to compare
            tgt = []
            for x in d:
                nonan = np.compress(~np.isnan(x), x)
                tgt.append(np.median(nonan, overwrite_input=True))

            assert_array_equal(np.nanmedian(d, axis=-1), tgt) 

def test_result_values(self):
            tgt = [np.median(d) for d in _rdat]
            res = np.nanmedian(_ndat, axis=1)
            assert_almost_equal(res, tgt) 

def test_empty(self):
        mat = np.zeros((0, 3))
        for axis in [0, None]:
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter('always')
                assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
                assert_(len(w) == 1)
                assert_(issubclass(w[0].category, RuntimeWarning))
        for axis in [1]:
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter('always')
                assert_equal(np.nanmedian(mat, axis=axis), np.zeros([]))
                assert_(len(w) == 0) 

def test_scalar(self):
        assert_(np.nanmedian(0.) == 0.) 

def test_nanmedian(self):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter("ignore", RuntimeWarning)
            self.check_funs(nanops.nanmedian, np.median, allow_complex=False,
                            allow_str=False, allow_date=False,
                            allow_tdelta=True, allow_obj='convert')