Python numpy.quantile() Examples

def pollution_confounded_propensity(intervention, untreated_runs, treatment_bias):
    """Probability of treating each unit.

    To generate confounding, we are more likely to treat worlds with high pollution.
    """

    def persistent_pollution(run):
        return run[intervention.time].persistent_pollution

    pollution = [persistent_pollution(run) for run in untreated_runs]
    upper_quantile = np.quantile(pollution, 0.9)

    def treatment_prob(idx):
        if pollution[idx] > upper_quantile:
            return treatment_bias
        return 1.0 - treatment_bias

    return np.array([treatment_prob(idx) for idx in range(len(untreated_runs))])


# pylint: disable-msg=invalid-name
#: An observational experiment with confounding. Polluted states are more likely to be treated. 

def find_best_eps(data, q=0.05):
    """
    Find best maximal distance (eps) between dots for DBSCAN clustering.

    Parameters
    -------
    data: pd.DataFrame
        Dataframe with features for clustering indexed as in ``retention_config.index_col``
    q: float, optional
        Quantile of nearest neighbor positive distance between dots. The value of it will be an eps. Default: ``0.05``

    Returns
    -------
    Optimal eps

    Return type
    -------
    Float
    """
    nn = NearestNeighbors()
    nn.fit(data)
    dist = nn.kneighbors()[0]
    dist = dist.flatten()
    dist = dist[dist > 0]
    return np.quantile(dist, q) 

def setSymColormap(self):
        cmap = {'ticks':
                [[0., (0, 0, 0, 255)],
                 [1e-3, (106, 0, 31, 255)],
                 [.5, (255, 255, 255, 255)],
                 [1., (8, 54, 104, 255)]],
                'mode': 'rgb'}
        cmap = {'ticks':
                [[0., (0, 0, 0)],
                 [1e-3, (172, 56, 56)],
                 [.5, (255, 255, 255)],
                 [1., (51, 53, 120)]],
                'mode': 'rgb'}

        relevant_data = num.abs(self._plot.data[num.isfinite(self._plot.data)])
        if num.any(relevant_data):
            lvl_max = num.quantile(relevant_data, .999)
        else:
            lvl_max = 1.

        self.gradient.restoreState(cmap)
        self.setLevels(-lvl_max, lvl_max) 

def trimean(self, data):
        """
        I'm exposing this as a public method because
        the trimean is not implemented in enough packages.
        
        Formula:
        (25th percentile + 2*50th percentile + 75th percentile)/4
        
        Parameters
        ----------
        data : array-like
          an iterable, either a list or a numpy array

        Returns
        -------
        the trimean: float
        """
        q1 = np.quantile(data, 0.25)
        q3 = np.quantile(data, 0.75)
        median = np.median(data)
        return (q1 + 2*median + q3)/4 

def trimean(self, data):
        """
        I'm exposing this as a public method because
        the trimean is not implemented in enough packages.
        
        Formula:
        (25th percentile + 2*50th percentile + 75th percentile)/4
        
        Parameters
        ----------
        data : array-like
          an iterable, either a list or a numpy array

        Returns
        -------
        the trimean: float
        """
        q1 = np.quantile(data, 0.25)
        q3 = np.quantile(data, 0.75)
        median = np.median(data)
        return (q1 + 2*median + q3)/4 

def prepare_confusion_mat(self, labels, scores, add_to_end=True, ):
        sorted_labels, sorted_scores = sort_score_and_label(labels, scores)

        score_threshold, cuts = None, None

        if self.cut_method == 'step':
            score_threshold, cuts = ThresholdCutter.cut_by_step(sorted_scores, steps=0.01)
            if add_to_end:
                score_threshold.append(min(score_threshold) - 0.001)
                cuts.append(1)

        elif self.cut_method == 'quantile':
            score_threshold = ThresholdCutter.cut_by_quantile(sorted_scores, remove_duplicate=self.remove_duplicate)
            score_threshold = list(np.flip(score_threshold))

        confusion_mat = ConfusionMatrix.compute(sorted_labels, sorted_scores, score_threshold,
                                                ret=['tp', 'fp', 'fn', 'tn'])

        return confusion_mat, score_threshold, cuts 

def __init__(self, recording, scale=1.0, median=0.0, q1=0.01, q2=0.99, seed=0):
        if not isinstance(recording, RecordingExtractor):
            raise ValueError("'recording' must be a RecordingExtractor")
        self._recording = recording

        random_data = self._get_random_data_for_scaling(seed=seed).ravel()
        loc_q1, pre_median, loc_q2 = np.quantile(random_data, q=[q1, 0.5, q2])
        pre_scale = abs(loc_q2 - loc_q1)

        self._scalar = scale / pre_scale
        self._offset = median - pre_median * self._scalar
        RecordingExtractor.__init__(self)
        self.copy_channel_properties(recording=self._recording)
        self.is_filtered = self._recording.is_filtered

        self._kwargs = {'recording': recording.make_serialized_dict(), 'scale': scale, 'median': median,
                        'q1': q1, 'q2': q2, 'seed': seed} 

def quantile_sorted(sorted_arr, quantile):
    # For small arrays (less than about 4000 items) np.quantile is significantly
    # slower than sorting the array and picking the quantile out by index. Computing
    # quantiles this way significantly improves performance for computing
    # trip time stats across all stops.

    max_index = len(sorted_arr) - 1
    quantile_index = max_index * quantile
    quantile_index_int = int(quantile_index)
    quantile_index_fractional = quantile_index - quantile_index_int

    quantile_lower = sorted_arr[quantile_index_int]
    if quantile_index_fractional > 0:
        quantile_upper = sorted_arr[quantile_index_int + 1]
        return quantile_lower + (quantile_upper - quantile_lower) * quantile_index_fractional
    else:
        return quantile_lower 

def get_tolerance(self,g):
        """
        Parameters
        ----------
        g: integer
           generation number of the ABC-SMC/MNN algorithm
        """
        # choose the tolerance given the generation number and how q and tol are defined
        if g == 0:
            if not hasattr(self.tol, "__len__"):
                return self.tol
            else:
                return self.tol[0]
        else:
            if self.q is not None:
                return np.quantile(self.dist,self.q)
            else:
                return self.tol[g] 

def test_wrap_ufunc_output(quantile, arg):
    ary = np.random.randn(4, 100)
    n_output = len(quantile)
    if arg:
        res = wrap_xarray_ufunc(
            np.quantile, ary, ufunc_kwargs={"n_output": n_output}, func_args=(quantile,)
        )
    else:
        if n_output == 1:
            res = wrap_xarray_ufunc(np.quantile, ary, func_kwargs={"q": quantile})
        else:
            res = wrap_xarray_ufunc(
                np.quantile, ary, ufunc_kwargs={"n_output": n_output}, func_kwargs={"q": quantile}
            )
    if n_output == 1:
        assert not isinstance(res, tuple)
    else:
        assert isinstance(res, tuple)
        assert len(res) == n_output 

def visit_quantile(transform: alt.QuantileTransform, df: pd.DataFrame) -> pd.DataFrame:
    transform = transform.to_dict()
    quantile = transform["quantile"]
    groupby = transform.get("groupby")
    pname, vname = transform.get("as", ["prob", "value"])
    probs = transform.get("probs")
    if probs is None:
        step = transform.get("step", 0.01)
        probs = np.arange(0.5 * step, 1.0, step)

    def qq(s: pd.Series) -> pd.DataFrame:
        return pd.DataFrame({pname: probs, vname: np.quantile(s, probs)})

    if groupby:
        return (
            df.groupby(groupby)[quantile]
            .apply(qq)
            .reset_index(groupby)
            .reset_index(drop=True)
        )

    else:
        return qq(df[quantile]).reset_index(drop=True) 

def report(self):
        est = self
        print(est.name, "max", np.max(est.errs), "99th",
              np.quantile(est.errs, 0.99), "95th", np.quantile(est.errs, 0.95),
              "median", np.quantile(est.errs, 0.5), "time_ms",
              np.mean(est.query_dur_ms)) 

def ReportEsts(estimators):
    v = -1
    for est in estimators:
        print(est.name, 'max', np.max(est.errs), '99th',
              np.quantile(est.errs, 0.99), '95th', np.quantile(est.errs, 0.95),
              'median', np.quantile(est.errs, 0.5))
        v = max(v, np.max(est.errs))
    return v 

def mediation_propensity_scores(intervention, untreated_runs):
    """Probability of treating each unit.

    Units with the largest populations are more likely to be treated.
    """
    populations = [run[intervention.time].population for run in untreated_runs]
    upper_quantile = np.quantile(populations, 0.9)
    propensities = 0.05 * np.ones(len(untreated_runs))
    propensities[populations > upper_quantile] = 0.9
    return propensities 

def bootstrap_sample_ate_ci(self, num_bootstrap_samples=2000, alpha=0.05):
        """Bootstrap a (1-alpha)% confidence interval for the sample ate."""
        means = []
        for _ in range(num_bootstrap_samples):
            sample = np.random.choice(
                self.true_effects, size=len(self.true_effects), replace=True
            )
            means.append(np.mean(sample))
        lower_tail, upper_tail = alpha / 2.0, 1.0 - alpha / 2.0
        return (np.quantile(means, lower_tail), np.quantile(means, upper_tail)) 

def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 

def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.quantile(x, 0), 0.)
        assert_equal(np.quantile(x, 1), 3.5)
        assert_equal(np.quantile(x, 0.5), 1.75) 

def test_no_p_overwrite(self):
        # this is worth retesting, because quantile does not make a copy
        p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
        p = p0.copy()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0)

        p0 = p0.tolist()
        p = p.tolist()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0) 

def detect_iris(frame, marks, side='left'):
    """
    return:
       x: the x coordinate of the iris.
       y: the y coordinate of the iris.
       x_rate: how much the iris is toward the left. 0 means totally left and 1 is totally right.
       y_rate: how much the iris is toward the top. 0 means totally top and 1 is totally bottom.
    """
    mask = np.full(frame.shape[:2], 255, np.uint8)
    if side == 'left':
        region = marks[36:42].astype(np.int32)
    elif side == 'right':
        region = marks[42:48].astype(np.int32)
    try:
        cv2.fillPoly(mask, [region], (0, 0, 0))
        eye = cv2.bitwise_not(frame, frame.copy(), mask=mask)
        # Cropping on the eye
        margin = 4
        min_x = np.min(region[:, 0]) - margin
        max_x = np.max(region[:, 0]) + margin
        min_y = np.min(region[:, 1]) - margin
        max_y = np.max(region[:, 1]) + margin

        eye = eye[min_y:max_y, min_x:max_x]
        eye = cv2.cvtColor(eye, cv2.COLOR_RGB2GRAY)

        eye_binarized = cv2.threshold(eye, np.quantile(eye, 0.2), 255, cv2.THRESH_BINARY)[1]
        contours, _ = cv2.findContours(eye_binarized, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
        contours = sorted(contours, key=cv2.contourArea)
        moments = cv2.moments(contours[-2])
        x = int(moments['m10'] / moments['m00']) + min_x
        y = int(moments['m01'] / moments['m00']) + min_y
        return x, y, (x-min_x-margin)/(max_x-min_x-2*margin), (y-min_y-margin)/(max_y-min_y-2*margin)
    except:
        return 0, 0, 0.5, 0.5 

def tmrca(ts):
    """
    Time to most recent common ancestor of sample, aka tree height.
    """
    tmrcas = [tree.time(tree.root) for tree in ts.trees()]
    min_, median, max_ = np.quantile(tmrcas, (0, 0.5, 1))
    return {"min(tmrca)": min_,
            "median(tmrca)": median,
            "max(tmrca)": max_,
            } 

def execute(cls, ctx, op):
        inputs, device_id, xp = as_same_device(
            [ctx[inp.key] for inp in op.inputs], device=op.device, ret_extra=True)
        a = inputs[0]
        out = inputs[-1].copy() if op.out is not None else None

        with device(device_id):
            ctx[op.outputs[0].key] = xp.quantile(a, q=op.q, axis=op.axis, out=out,
                                                 interpolation=op.interpolation,
                                                 keepdims=op.keepdims) 

def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 

def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]:
        if len(archive) < self.max_len:
            return archive
        quantiles: tp.Dict[str, float] = {}
        threshold = float(self.min_len) / len(archive)
        names = ["optimistic", "pessimistic", "average"]
        for name in names:
            quantiles[name] = np.quantile([v.get_estimation(name) for v in archive.values()], threshold, interpolation="lower")
        new_archive: Archive[MultiValue] = Archive()
        new_archive.bytesdict = {b: v for b, v in archive.bytesdict.items() if any(v.get_estimation(n) <= quantiles[n] for n in names)}
        return new_archive 

def detect_poison(self, **kwargs):
        """
        Returns poison detected and a report.
        :return: (report, is_clean_lst):
                where a report is None (for future ART compatibility)
                where is_clean is a list, where is_clean_lst[i]=1 means that x_train[i]
                there is clean and is_clean_lst[i]=0, means that x_train[i] was classified as poison.
        """

        self.set_params(**kwargs)

        n_classes = self.classifier.nb_classes()
        nb_layers = len(self.classifier.layer_names)

        features_x_poisoned = self.classifier.get_activations(
            self.x_train, layer=nb_layers - 1, batch_size=self.batch_size
        )

        features_split = SpectralSignatureDefense.split_by_class(
            features_x_poisoned, self.y_train, n_classes
        )
        keep_by_class = []
        for idx, feature in enumerate(features_split):
            score = SpectralSignatureDefense.spectral_signature_scores(feature)
            score_cutoff = np.quantile(
                score, max(1 - self.eps_multiplier * self.ub_pct_poison, 0.0)
            )
            keep_by_class.append(score < score_cutoff)

        base_indices_by_class = SpectralSignatureDefense.split_by_class(
            np.arange(self.y_train.shape[0]), self.y_train, 10
        )
        is_clean_lst = np.zeros_like(self.y_train, dtype=np.int)

        for keep_booleans, indices in zip(keep_by_class, base_indices_by_class):
            for keep_boolean, idx in zip(keep_booleans, indices):
                if keep_boolean:
                    is_clean_lst[idx] = 1

        return None, is_clean_lst 

def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 

def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.quantile(x, 0), 0.)
        assert_equal(np.quantile(x, 1), 3.5)
        assert_equal(np.quantile(x, 0.5), 1.75) 

def test_no_p_overwrite(self):
        # this is worth retesting, because quantile does not make a copy
        p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
        p = p0.copy()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0)

        p0 = p0.tolist()
        p = p.tolist()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0) 

def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None,
              **kwargs) -> float:
        # label actuals as 1 or 0
        lb = LabelEncoder()
        labels = lb.fit_transform(labels)
        actual = lb.transform(actual)

        # probability of predicted response likelihood above which we'll send a letter
        cutoff = np.quantile(predicted, self._quantile)
        # print("cutoff: %f" % cutoff)

        # whom we'll send letter to
        selected = (predicted >= cutoff).ravel()
        num_letters = np.count_nonzero(selected)
        # print("number of letters: %d" % num_letters)

        # compute cost and reward
        cost = num_letters * self._cost  # each letter costs _cost
        reward = np.count_nonzero(actual[selected] == 1) * self._reward  # each true positive leads to _reward
        # print("cost: %f" % cost)
        # print("reward: %f" % reward)

        # compute total net income
        net_income = reward - cost
        # print("net_income: %f" % net_income)

        # return mean profit per letter sent
        return 0 if num_letters == 0 else net_income / num_letters 

def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None,
              **kwargs) -> float:
        cutoff = np.quantile(predicted, 0.9)
        which = (predicted >= cutoff).ravel()
        return float(np.median(np.abs(actual[which] - predicted[which]))) 

def get_parameter_choices():
        return {"quantile": [0.01, 0.001, 0.05]}