Python numpy.nanmean() Examples

def compute_average_mAP_N_for_characteristic(sensitivity_analysis, characteristic_name):
    gt_by_characteristic = sensitivity_analysis.ground_truth.groupby(characteristic_name)
    average_mAP_n_by_characteristic_value = OrderedDict()
    
    for characteristic_value, this_characteristic_gt in gt_by_characteristic:
        ap = np.nan*np.zeros(len(sensitivity_analysis.activity_index))
        gt_by_cls = this_characteristic_gt.groupby('label')
        pred_by_cls = sensitivity_analysis.prediction.groupby('label')
        for cls in sensitivity_analysis.activity_index.values():
            this_cls_pred = pred_by_cls.get_group(cls).sort_values(by='score',ascending=False)
            try:
                this_cls_gt = gt_by_cls.get_group(cls)
            except:
                continue
            gt_id_to_keep = np.append(this_cls_gt['gt-id'].values, [np.nan])
        
            for tidx, tiou in enumerate(sensitivity_analysis.tiou_thresholds):
                this_cls_pred = this_cls_pred[this_cls_pred[sensitivity_analysis.matched_gt_id_cols[tidx]].isin(gt_id_to_keep)]
            
            ap[cls] = compute_mAP_N(sensitivity_analysis,this_cls_pred,this_cls_gt)

        average_mAP_n_by_characteristic_value[characteristic_value] = np.nanmean(ap)

    return average_mAP_n_by_characteristic_value 

def _nanquantile_unchecked(a, q, axis=None, out=None, overwrite_input=False,
                           interpolation='linear', keepdims=np._NoValue):
    """Assumes that q is in [0, 1], and is an ndarray"""
    # apply_along_axis in _nanpercentile doesn't handle empty arrays well,
    # so deal them upfront
    if a.size == 0:
        return np.nanmean(a, axis, out=out, keepdims=keepdims)

    r, k = function_base._ureduce(
        a, func=_nanquantile_ureduce_func, q=q, axis=axis, out=out,
        overwrite_input=overwrite_input, interpolation=interpolation
    )
    if keepdims and keepdims is not np._NoValue:
        return r.reshape(q.shape + k)
    else:
        return r 

def execute_agg(cls, ctx, op):
        axis = cls.get_axis(op.axis)

        a = ctx[op.inputs[0].key]
        if not isinstance(a, (list, tuple)):
            (inp,), device_id, xp = as_same_device(
                [a], device=op.device, ret_extra=True)

            with device(device_id):
                ctx[op.outputs[0].key] = xp.nanmean(inp, axis=axis, dtype=op.dtype,
                                                    keepdims=bool(op.keepdims))
        else:
            (_data, _count), device_id, xp = as_same_device(
                a, device=op.device, ret_extra=True)

            with device(device_id):
                chunk_count = xp.sum(_count, axis=axis, dtype=op.dtype,
                                     keepdims=bool(op.keepdims))
                chunk_sum = xp.sum(_data, axis=axis, dtype=op.dtype,
                                   keepdims=bool(op.keepdims))
                ctx[op.outputs[0].key] = xp.true_divide(chunk_sum, chunk_count,
                                                        dtype=op.dtype) 

def scores(label_trues, label_preds, n_class):
    hist = np.zeros((n_class, n_class))
    for lt, lp in zip(label_trues, label_preds):
        hist += _fast_hist(lt.flatten(), lp.flatten(), n_class)
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    valid = hist.sum(axis=1) > 0  # added
    mean_iu = np.nanmean(iu[valid])
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    cls_iu = dict(zip(range(n_class), iu))

    return {
        "Overall Acc": acc,
        "Mean Acc": acc_cls,
        "FreqW Acc": fwavacc,
        "Mean IoU": mean_iu,
        "Class IoU": cls_iu,
    } 

def scores(label_trues, label_preds, n_class):
    hist = np.zeros((n_class, n_class))
    for lt, lp in zip(label_trues, label_preds):
        if(lt.size > 0):
            hist += _fast_hist(lt.flatten(), lp.flatten(), n_class)
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    cls_iu = dict(zip(range(n_class), iu))

    return {
        "Overall Acc": acc,
        "Mean Acc": acc_cls,
        "FreqW Acc": fwavacc,
        "Mean IoU": mean_iu,
    }, cls_iu 

def scores(label_trues, label_preds, n_class):
    hist = np.zeros((n_class, n_class))
    for lt, lp in zip(label_trues, label_preds):
        if(lt.size > 0):
            hist += _fast_hist(lt.flatten(), lp.flatten(), n_class)
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    cls_iu = dict(zip(range(n_class), iu))

    return {
        "Overall Acc": acc,
        "Mean Acc": acc_cls,
        "FreqW Acc": fwavacc,
        "Mean IoU": mean_iu,
    }, cls_iu 

def eval_detection_voc(pred_boxlists, gt_boxlists, iou_thresh=0.5, use_07_metric=False):
    """Evaluate on voc dataset.
    Args:
        pred_boxlists(list[BoxList]): pred boxlist, has labels and scores fields.
        gt_boxlists(list[BoxList]): ground truth boxlist, has labels field.
        iou_thresh: iou thresh
        use_07_metric: boolean
    Returns:
        dict represents the results
    """
    assert len(gt_boxlists) == len(
        pred_boxlists
    ), "Length of gt and pred lists need to be same."
    prec, rec, n_tp, n_fp, n_pos = calc_detection_voc_prec_rec(
        pred_boxlists=pred_boxlists, gt_boxlists=gt_boxlists, iou_thresh=iou_thresh
    )
    ap = calc_detection_voc_ap(prec, rec, use_07_metric=use_07_metric)
    prec = {k: v.tolist() for k, v in prec.items()}
    rec = {k: v.tolist() for k, v in rec.items()}
    res = [{"ap": ap[k], "class_id": k, "precisions": prec[k], "recalls": rec[k],
            "n_tp": n_tp[k], "n_fp": n_fp[k], "n_positives": n_pos[k]} for k in ap.keys()]
    return res, np.nanmean(ap.values()) 

def print_depth_stats(mids, err):
    shape_ids = np.unique([m[0] for m in mids])
    serr = dict()
    for sid in shape_ids:
        serr[sid] = []

    for ex, e in enumerate(err):
        serr[mids[ex][0]].append(e)

    table = []
    smean = []
    for s in serr:
        sm = np.nanmean(serr[s])
        table.append([s, sm])
        smean.append(sm)
    table.append(['Mean', np.nanmean(smean)])
    ptable = tabulate(table, headers=['SID', 'L1 error'], floatfmt=".4f")
    return smean, ptable 

def _signal_synchrony_correlation(signal1, signal2, window_size, center=False):
    """Calculates pairwise rolling correlation at each time. Grabs the upper triangle, at each timepoints.

    - window: window size of rolling corr in samples
    - center: whether to center result (Default: False, so correlation values are listed on the right.)

    """
    data = pd.DataFrame({"y1": signal1, "y2": signal2})

    rolled = data.rolling(window=window_size, center=center).corr()
    synchrony = rolled["y1"].loc[rolled.index.get_level_values(1) == "y2"].values

    # Realign
    synchrony = np.append(synchrony[int(window_size / 2) :], np.full(int(window_size / 2), np.nan))
    synchrony[np.isnan(synchrony)] = np.nanmean(synchrony)

    return synchrony 

def test_select_confounds(confounds_file, selected_confounds, nan_confounds,
                          expanded_confounds):
    import pandas as pd
    import numpy as np

    confounds_df = pd.read_csv(str(confounds_file), sep='\t', na_values='n/a')

    res_df = _select_confounds(str(confounds_file), selected_confounds)

    # check if the correct columns are selected
    assert set(expanded_confounds) == set(res_df.columns)

    # check if nans are being imputed when expected
    if nan_confounds:
        for nan_c in nan_confounds:
            vals = confounds_df[nan_c].values
            expected_result = np.nanmean(vals[vals != 0])
            assert res_df[nan_c][0] == expected_result 

def print_iou(iu, mean_pixel_acc, class_names=None, show_no_back=False):
    n = iu.size
    lines = []
    for i in range(n):
        if class_names is None:
            cls = 'Class %d:' % (i + 1)
        else:
            cls = '%d %s' % (i + 1, class_names[i])
        # lines.append('%-8s: %.3f%%' % (cls, iu[i] * 100))
    mean_IU = np.nanmean(iu)
    mean_IU_no_back = np.nanmean(iu[1:])
    if show_no_back:
        lines.append('mean_IU: %.3f%% || mean_IU_no_back: %.3f%% || mean_pixel_acc: %.3f%%' % (
            mean_IU * 100, mean_IU_no_back * 100, mean_pixel_acc * 100))
    else:
        lines.append('mean_IU: %.3f%% || mean_pixel_acc: %.3f%%' % (mean_IU * 100, mean_pixel_acc * 100))
    lines.append('=================================================')
    line = "\n".join(lines)

    print(line) 

def mape(y_pred, y_test):
    r"""
    The Mean Absolute Percentage Error (MAPE)

    The MAPE is computed as the mean of the absolute value of the relative
    error in percent, i.e.:

    .. math::

        \text{MAPE}(\mathbf{y}, \mathbf{y}_{true}) =
        \frac{100\%}{n}\sum_{i = 0}^n \frac{|y_{\text{pred},i} - y_{\text{true},i}|}
             {|y_{\text{true},i}|}

    Arguments:

        y_pred(numpy.array): The predicted scalar values.

        y_test(numpy.array): The true values.

    Returns:

        The MAPE for the given predictions.

    """
    return np.nanmean(100.0 * np.abs(y_test - y_pred.ravel()) / np.abs(y_test).ravel()) 

def eval_detection_voc(pred_boxlists, gt_boxlists, iou_thresh=0.5, use_07_metric=False):
    """Evaluate on voc dataset.
    Args:
        pred_boxlists(list[BoxList]): pred boxlist, has labels and scores fields.
        gt_boxlists(list[BoxList]): ground truth boxlist, has labels field.
        iou_thresh: iou thresh
        use_07_metric: boolean
    Returns:
        dict represents the results
    """
    assert len(gt_boxlists) == len(
        pred_boxlists
    ), "Length of gt and pred lists need to be same."
    prec, rec = calc_detection_voc_prec_rec(
        pred_boxlists=pred_boxlists, gt_boxlists=gt_boxlists, iou_thresh=iou_thresh
    )
    ap = calc_detection_voc_ap(prec, rec, use_07_metric=use_07_metric)
    return {"ap": ap, "map": np.nanmean(ap)} 

def segmentation_scores(label_trues, label_preds, n_class):
    """Returns accuracy score evaluation result.
      - overall accuracy
      - mean accuracy
      - mean IU
      - fwavacc
    """
    hist = np.zeros((n_class, n_class))
    for lt, lp in zip(label_trues, label_preds):
        hist += _fast_hist(lt.flatten(), lp.flatten(), n_class)
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()

    return {'overall_acc': acc,
            'mean_acc': acc_cls,
            'freq_w_acc': fwavacc,
            'mean_iou': mean_iu} 

def cal_scores(hist):
    n_class = settings.N_CLASSES
    #acc = np.diag(hist).sum() / hist.sum()
    #acc_cls = np.diag(hist) / hist.sum(axis=1)
    #acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    cls_iu = dict(zip(label_names, iu))

    return {
        #"pAcc": acc,
        #"mAcc": acc_cls,
        "fIoU": fwavacc,
        "mIoU": mean_iu,
    }#, cls_iu 

def cal_scores(hist):
    n_class = settings.N_CLASSES
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    cls_iu = dict(zip(label_names, iu))

    return {
        'pAcc': acc,
        'mAcc': acc_cls,
        'fIoU': fwavacc,
        'mIoU': mean_iu,
    }, cls_iu 

def get_scores(self):
        """Returns accuracy score evaluation result.
            - overall accuracy
            - mean accuracy
            - mean IU
            - fwavacc
        """
        hist = self.confusion_matrix
        acc = np.diag(hist).sum() / hist.sum()
        acc_cls = np.diag(hist) / hist.sum(axis=1)
        acc_cls = np.nanmean(acc_cls)
        iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
        mean_iu = np.nanmean(iu)
        freq = hist.sum(axis=1) / hist.sum()
        fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
        cls_iu = dict(zip(range(self.n_classes), iu))

        return {'Overall Acc: \t': acc,
                'Mean Acc : \t': acc_cls,
                'FreqW Acc : \t': fwavacc,
                'Mean IoU : \t': mean_iu,}, cls_iu 

def _recmat_smooth(presented, recalled, features, distance, match):

    if match == 'best':
        func = np.argmax
    elif match == 'smooth':
        func = np.nanmean

    simmtx = _similarity_smooth(presented, recalled, features, distance)


    if match == 'best':
        recmat = np.atleast_3d([func(s, 1) for s in simmtx]).astype(np.float64)
        recmat+=1
        recmat[np.isnan(simmtx).any(2)]=np.nan
    elif match == 'smooth':
        recmat = np.atleast_3d([func(s, 0) for s in simmtx]).astype(np.float64)


    return recmat 

def _similarity_smooth(presented, recalled, features, distance):
    lists = presented.index.get_values()
    res = np.empty((len(lists), len(features), recalled.iloc[0].shape[0], presented.iloc[0].shape[0]))*np.nan
    for li, l in enumerate(lists):
        p_list = presented.loc[l]
        r_list = recalled.loc[l]
        for i, feature in enumerate(features):
            get_feature = lambda x: np.array(x[feature]) if np.array(pd.notna(x['item'])).any() else np.nan
            p = np.vstack(p_list.apply(get_feature).get_values())
            r = r_list.dropna().apply(get_feature).get_values()
            r = np.vstack(list(filter(lambda x: x is not np.nan, r)))
            tmp = 1 - cdist(r, p, distance)
            res[li, i, :tmp.shape[0], :] =  tmp
    if distance == 'correlation':
        return np.nanmean(res, 1)
    else:
        return np.mean(res, 1) 

def _get_weight_best(egg, feature, distdict, permute, n_perms, distance):

    if permute:
        return _permute(egg, feature, distdict, _get_weight_best, n_perms)

    rec = list(egg.get_rec_items().values[0])
    if len(rec) <= 2:
        warnings.warn('Not enough recalls to compute fingerprint, returning default'
              'fingerprint.. (everything is .5)')
        return np.nan

    distmat = get_distmat(egg, feature, distdict)
    matchmat = get_match(egg, feature, distdict)

    ranks = []
    for i in range(len(rec)-1):
        cdx, ndx = np.argmin(matchmat[i, :]), np.argmin(matchmat[i+1, :])
        dists = distmat[cdx, :]
        di = dists[ndx]
        dists_filt = np.array([dist for idx, dist in enumerate(dists)])
        ranks.append(np.mean(np.where(np.sort(dists_filt)[::-1] == di)[0]+1) / len(dists_filt))
    return np.nanmean(ranks) 

def _get_weight_smooth(egg, feature, distdict, permute, n_perms, distance):

    if permute:
        return _permute(egg, feature, distdict, _get_weight_smooth, n_perms)

    rec = list(egg.get_rec_items().values[0])
    if len(rec) <= 2:
        warnings.warn('Not enough recalls to compute fingerprint, returning default'
              'fingerprint.. (everything is .5)')
        return np.nan

    distmat = get_distmat(egg, feature, distdict)
    matchmat = get_match(egg, feature, distdict)

    ranks = []
    for i in range(len(rec)-1):
        cdx, ndx = np.argmin(matchmat[i, :]), np.argmin(matchmat[i+1, :])
        dists = distmat[cdx, :]
        di = dists[ndx]
        dists_filt = np.array([dist for idx, dist in enumerate(dists)])
        ranks.append(np.mean(np.where(np.sort(dists_filt)[::-1] == di)[0]+1) / len(dists_filt))
    return np.nanmean(ranks) 

def _nanquantile_unchecked(a, q, axis=None, out=None, overwrite_input=False,
                           interpolation='linear', keepdims=np._NoValue):
    """Assumes that q is in [0, 1], and is an ndarray"""
    # apply_along_axis in _nanpercentile doesn't handle empty arrays well,
    # so deal them upfront
    if a.size == 0:
        return np.nanmean(a, axis, out=out, keepdims=keepdims)

    r, k = function_base._ureduce(
        a, func=_nanquantile_ureduce_func, q=q, axis=axis, out=out,
        overwrite_input=overwrite_input, interpolation=interpolation
    )
    if keepdims and keepdims is not np._NoValue:
        return r.reshape(q.shape + k)
    else:
        return r 

def eval_detection_voc(pred_boxlists, gt_boxlists, iou_thresh=0.5, use_07_metric=False):
    """Evaluate on voc dataset.
    Args:
        pred_boxlists(list[BoxList]): pred boxlist, has labels and scores fields.
        gt_boxlists(list[BoxList]): ground truth boxlist, has labels field.
        iou_thresh: iou thresh
        use_07_metric: boolean
    Returns:
        dict represents the results
    """
    assert len(gt_boxlists) == len(
        pred_boxlists
    ), "Length of gt and pred lists need to be same."
    prec, rec = calc_detection_voc_prec_rec(
        pred_boxlists=pred_boxlists, gt_boxlists=gt_boxlists, iou_thresh=iou_thresh
    )
    ap = calc_detection_voc_ap(prec, rec, use_07_metric=use_07_metric)
    return {"ap": ap, "map": np.nanmean(ap)} 

def get_scores(hist):
    # Mean pixel accuracy
    acc = np.diag(hist).sum() / (hist.sum() + 1e-12)

    # Per class accuracy
    cl_acc = np.diag(hist) / (hist.sum(1) + 1e-12)

    # Per class IoU
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist) + 1e-12)

    return acc, np.nanmean(cl_acc), np.nanmean(iu), cl_acc, iu 

def test_complete_features():

    # Test with use_complete=True
    X = np.array([
        [0,      np.nan,    0,       np.nan],
        [1,      1,         1,       np.nan],
        [2,      2,         np.nan,  2],
        [3,      3,         3,       3],
        [4,      4,         4,       4],
        [5,      5,         5,       5],
        [6,      6,         6,       6],
        [np.nan, 7,         7,       7]
    ])

    r0c1 = np.mean(X[1:6, 1])
    r0c3 = np.mean(X[2:-1, -1])
    r1c3 = np.mean(X[2:-1, -1])
    r2c2 = np.nanmean(X[:6, 2])
    r7c0 = np.mean(X[2:-1, 0])

    X_imputed = np.array([
        [0,     r0c1,   0,    r0c3],
        [1,     1,      1,    r1c3],
        [2,     2,      r2c2, 2],
        [3,     3,      3,    3],
        [4,     4,      4,    4],
        [5,     5,      5,    5],
        [6,     6,      6,    6],
        [r7c0,  7,      7,    7]
    ])

    imputer_comp = KNNImputer()
    assert_array_almost_equal(imputer_comp.fit_transform(X), X_imputed) 

def test_knn_imputation_zero_p2():
    # Test with an imputable matrix and also compare with missing_values="NaN"
    X_zero = np.array([
        [1, 0, 1, 1, 1.],
        [2, 2, 2, 2, 2],
        [3, 3, 3, 3, 0],
        [6, 6, 0, 6, 6],
    ])

    X_nan = np.array([
        [1, np.nan, 1,      1,      1.],
        [2, 2,      2,      2,      2],
        [3, 3,      3,      3,      np.nan],
        [6, 6,      np.nan, 6,      6],
    ])
    statistics_mean = np.nanmean(X_nan, axis=0)

    X_imputed = np.array([
        [1, 2.5,    1,   1, 1.],
        [2, 2,      2,   2, 2],
        [3, 3,      3,   3, 1.5],
        [6, 6,      2.5, 6, 6],
    ])

    imputer_zero = KNNImputer(missing_values=0, n_neighbors=2,
                              weights="uniform")

    imputer_nan = KNNImputer(missing_values="NaN",
                             n_neighbors=2,
                             weights="uniform")

    assert_array_equal(imputer_zero.fit_transform(X_zero), X_imputed)
    assert_array_equal(imputer_zero.statistics_, statistics_mean)
    assert_array_equal(imputer_zero.fit_transform(X_zero),
                       imputer_nan.fit_transform(X_nan)) 

def getMetric(self):
        if self.nClasses == 20:
            hist = self.total_hist[0:19,0:19]
        else:
            hist = self.total_hist
        epsilon = 0.00000001

        overall_acc = np.diag(hist).sum() / (hist.sum() + epsilon)
        per_class_acc = np.diag(hist) / (hist.sum(1) + epsilon)
        per_class_iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist) + epsilon)
        mIou = np.nanmean(per_class_iu)

        return overall_acc, per_class_acc, per_class_iu, mIou 

def test_nanmean(self):
        tgt = np.mean(self.mat)
        for mat in self.integer_arrays():
            assert_equal(np.nanmean(mat), tgt) 

def eval_l1_err(pred, gt, mask=None, vis=False):
    pred = pred[:, 0, ...]
    bs, im_batch = pred.shape[0], pred.shape[1]
    if mask is None:
        nanmask = (gt < np.max(gt))
    range_mask = np.logical_and(pred > 2.0 - np.sqrt(3) * 0.5,
                                pred < 2.0 + np.sqrt(3) * 0.5)
    mask = np.logical_and(nanmask, range_mask)

    if vis:
        plt.subplot(5, 1, 1)
        vis_ims(mask)
        plt.title("Eval Mask")
        plt.subplot(5, 1, 2)
        vis_ims(pred / 10.0, mask=mask)
        plt.title("Pred")
        plt.subplot(5, 1, 3)
        vis_ims(gt / 10.0, mask=nanmask)
        plt.title("Gt")
        plt.subplot(5, 1, 4)
        vis_ims(np.logical_xor(mask, nanmask))
        plt.title("Gt Mask - Mask")
        plt.subplot(5, 1, 5)
        vis_ims(np.abs(pred - gt) / 10.0, mask=mask)
        plt.title("Masked L1 error")
        plt.show()

    l1_err = np.abs(pred - gt)
    l1_err_masked = np.ma.array(l1_err, mask=np.logical_not(mask))
    batch_err = []
    for b in range(bs):
        tmp = np.zeros((im_batch, ))
        for imb in range(im_batch):
            tmp[imb] = np.ma.median(l1_err_masked[b, imb])
        batch_err.append(np.nanmean(tmp))
    return batch_err 

def test_nanfunctions_matrices_general():
    # Check that it works and that type and
    # shape are preserved
    # 2018-04-29: moved here from core.tests.test_nanfunctions
    mat = np.matrix(np.eye(3))
    for f in (np.nanargmin, np.nanargmax, np.nansum, np.nanprod,
              np.nanmean, np.nanvar, np.nanstd):
        res = f(mat, axis=0)
        assert_(isinstance(res, np.matrix))
        assert_(res.shape == (1, 3))
        res = f(mat, axis=1)
        assert_(isinstance(res, np.matrix))
        assert_(res.shape == (3, 1))
        res = f(mat)
        assert_(np.isscalar(res))

    for f in np.nancumsum, np.nancumprod:
        res = f(mat, axis=0)
        assert_(isinstance(res, np.matrix))
        assert_(res.shape == (3, 3))
        res = f(mat, axis=1)
        assert_(isinstance(res, np.matrix))
        assert_(res.shape == (3, 3))
        res = f(mat)
        assert_(isinstance(res, np.matrix))
        assert_(res.shape == (1, 3*3))