Python numpy.quantile() Examples

The following are 30 code examples for showing how to use numpy.quantile(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module numpy , or try the search function .

Example 1
Project: whynot   Author: zykls   File: experiments.py    License: MIT License 6 votes vote down vote up
def pollution_confounded_propensity(intervention, untreated_runs, treatment_bias):
    """Probability of treating each unit.

    To generate confounding, we are more likely to treat worlds with high pollution.
    """

    def persistent_pollution(run):
        return run[intervention.time].persistent_pollution

    pollution = [persistent_pollution(run) for run in untreated_runs]
    upper_quantile = np.quantile(pollution, 0.9)

    def treatment_prob(idx):
        if pollution[idx] > upper_quantile:
            return treatment_bias
        return 1.0 - treatment_bias

    return np.array([treatment_prob(idx) for idx in range(len(untreated_runs))])


# pylint: disable-msg=invalid-name
#: An observational experiment with confounding. Polluted states are more likely to be treated. 
Example 2
Project: retentioneering-tools   Author: retentioneering   File: clustering.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def find_best_eps(data, q=0.05):
    """
    Find best maximal distance (eps) between dots for DBSCAN clustering.

    Parameters
    -------
    data: pd.DataFrame
        Dataframe with features for clustering indexed as in ``retention_config.index_col``
    q: float, optional
        Quantile of nearest neighbor positive distance between dots. The value of it will be an eps. Default: ``0.05``

    Returns
    -------
    Optimal eps

    Return type
    -------
    Float
    """
    nn = NearestNeighbors()
    nn.fit(data)
    dist = nn.kneighbors()[0]
    dist = dist.flatten()
    dist = dist[dist > 0]
    return np.quantile(dist, q) 
Example 3
Project: kite   Author: pyrocko   File: base.py    License: GNU General Public License v3.0 6 votes vote down vote up
def setSymColormap(self):
        cmap = {'ticks':
                [[0., (0, 0, 0, 255)],
                 [1e-3, (106, 0, 31, 255)],
                 [.5, (255, 255, 255, 255)],
                 [1., (8, 54, 104, 255)]],
                'mode': 'rgb'}
        cmap = {'ticks':
                [[0., (0, 0, 0)],
                 [1e-3, (172, 56, 56)],
                 [.5, (255, 255, 255)],
                 [1., (51, 53, 120)]],
                'mode': 'rgb'}

        relevant_data = num.abs(self._plot.data[num.isfinite(self._plot.data)])
        if num.any(relevant_data):
            lvl_max = num.quantile(relevant_data, .999)
        else:
            lvl_max = 1.

        self.gradient.restoreState(cmap)
        self.setLevels(-lvl_max, lvl_max) 
Example 4
Project: drifter_ml   Author: EricSchles   File: regression_tests.py    License: MIT License 6 votes vote down vote up
def trimean(self, data):
        """
        I'm exposing this as a public method because
        the trimean is not implemented in enough packages.
        
        Formula:
        (25th percentile + 2*50th percentile + 75th percentile)/4
        
        Parameters
        ----------
        data : array-like
          an iterable, either a list or a numpy array

        Returns
        -------
        the trimean: float
        """
        q1 = np.quantile(data, 0.25)
        q3 = np.quantile(data, 0.75)
        median = np.median(data)
        return (q1 + 2*median + q3)/4 
Example 5
Project: drifter_ml   Author: EricSchles   File: classification_tests.py    License: MIT License 6 votes vote down vote up
def trimean(self, data):
        """
        I'm exposing this as a public method because
        the trimean is not implemented in enough packages.
        
        Formula:
        (25th percentile + 2*50th percentile + 75th percentile)/4
        
        Parameters
        ----------
        data : array-like
          an iterable, either a list or a numpy array

        Returns
        -------
        the trimean: float
        """
        q1 = np.quantile(data, 0.25)
        q3 = np.quantile(data, 0.75)
        median = np.median(data)
        return (q1 + 2*median + q3)/4 
Example 6
Project: FATE   Author: FederatedAI   File: classification_metric.py    License: Apache License 2.0 6 votes vote down vote up
def prepare_confusion_mat(self, labels, scores, add_to_end=True, ):
        sorted_labels, sorted_scores = sort_score_and_label(labels, scores)

        score_threshold, cuts = None, None

        if self.cut_method == 'step':
            score_threshold, cuts = ThresholdCutter.cut_by_step(sorted_scores, steps=0.01)
            if add_to_end:
                score_threshold.append(min(score_threshold) - 0.001)
                cuts.append(1)

        elif self.cut_method == 'quantile':
            score_threshold = ThresholdCutter.cut_by_quantile(sorted_scores, remove_duplicate=self.remove_duplicate)
            score_threshold = list(np.flip(score_threshold))

        confusion_mat = ConfusionMatrix.compute(sorted_labels, sorted_scores, score_threshold,
                                                ret=['tp', 'fp', 'fn', 'tn'])

        return confusion_mat, score_threshold, cuts 
Example 7
Project: spiketoolkit   Author: SpikeInterface   File: normalize_by_quantile.py    License: MIT License 6 votes vote down vote up
def __init__(self, recording, scale=1.0, median=0.0, q1=0.01, q2=0.99, seed=0):
        if not isinstance(recording, RecordingExtractor):
            raise ValueError("'recording' must be a RecordingExtractor")
        self._recording = recording

        random_data = self._get_random_data_for_scaling(seed=seed).ravel()
        loc_q1, pre_median, loc_q2 = np.quantile(random_data, q=[q1, 0.5, q2])
        pre_scale = abs(loc_q2 - loc_q1)

        self._scalar = scale / pre_scale
        self._offset = median - pre_median * self._scalar
        RecordingExtractor.__init__(self)
        self.copy_channel_properties(recording=self._recording)
        self.is_filtered = self._recording.is_filtered

        self._kwargs = {'recording': recording.make_serialized_dict(), 'scale': scale, 'median': median,
                        'q1': q1, 'q2': q2, 'seed': seed} 
Example 8
Project: metrics-mvp   Author: trynmaps   File: util.py    License: MIT License 6 votes vote down vote up
def quantile_sorted(sorted_arr, quantile):
    # For small arrays (less than about 4000 items) np.quantile is significantly
    # slower than sorting the array and picking the quantile out by index. Computing
    # quantiles this way significantly improves performance for computing
    # trip time stats across all stops.

    max_index = len(sorted_arr) - 1
    quantile_index = max_index * quantile
    quantile_index_int = int(quantile_index)
    quantile_index_fractional = quantile_index - quantile_index_int

    quantile_lower = sorted_arr[quantile_index_int]
    if quantile_index_fractional > 0:
        quantile_upper = sorted_arr[quantile_index_int + 1]
        return quantile_lower + (quantile_upper - quantile_lower) * quantile_index_fractional
    else:
        return quantile_lower 
Example 9
Project: pygom   Author: publichealthengland   File: approximate_bayesian_computation.py    License: GNU General Public License v2.0 6 votes vote down vote up
def get_tolerance(self,g):
        """
        Parameters
        ----------
        g: integer
           generation number of the ABC-SMC/MNN algorithm
        """
        # choose the tolerance given the generation number and how q and tol are defined
        if g == 0:
            if not hasattr(self.tol, "__len__"):
                return self.tol
            else:
                return self.tol[0]
        else:
            if self.q is not None:
                return np.quantile(self.dist,self.q)
            else:
                return self.tol[g] 
Example 10
Project: arviz   Author: arviz-devs   File: test_stats_utils.py    License: Apache License 2.0 6 votes vote down vote up
def test_wrap_ufunc_output(quantile, arg):
    ary = np.random.randn(4, 100)
    n_output = len(quantile)
    if arg:
        res = wrap_xarray_ufunc(
            np.quantile, ary, ufunc_kwargs={"n_output": n_output}, func_args=(quantile,)
        )
    else:
        if n_output == 1:
            res = wrap_xarray_ufunc(np.quantile, ary, func_kwargs={"q": quantile})
        else:
            res = wrap_xarray_ufunc(
                np.quantile, ary, ufunc_kwargs={"n_output": n_output}, func_kwargs={"q": quantile}
            )
    if n_output == 1:
        assert not isinstance(res, tuple)
    else:
        assert isinstance(res, tuple)
        assert len(res) == n_output 
Example 11
Project: altair-transform   Author: altair-viz   File: quantile.py    License: MIT License 6 votes vote down vote up
def visit_quantile(transform: alt.QuantileTransform, df: pd.DataFrame) -> pd.DataFrame:
    transform = transform.to_dict()
    quantile = transform["quantile"]
    groupby = transform.get("groupby")
    pname, vname = transform.get("as", ["prob", "value"])
    probs = transform.get("probs")
    if probs is None:
        step = transform.get("step", 0.01)
        probs = np.arange(0.5 * step, 1.0, step)

    def qq(s: pd.Series) -> pd.DataFrame:
        return pd.DataFrame({pname: probs, vname: np.quantile(s, probs)})

    if groupby:
        return (
            df.groupby(groupby)[quantile]
            .apply(qq)
            .reset_index(groupby)
            .reset_index(drop=True)
        )

    else:
        return qq(df[quantile]).reset_index(drop=True) 
Example 12
Project: naru   Author: naru-project   File: estimators.py    License: Apache License 2.0 5 votes vote down vote up
def report(self):
        est = self
        print(est.name, "max", np.max(est.errs), "99th",
              np.quantile(est.errs, 0.99), "95th", np.quantile(est.errs, 0.95),
              "median", np.quantile(est.errs, 0.5), "time_ms",
              np.mean(est.query_dur_ms)) 
Example 13
Project: naru   Author: naru-project   File: eval_model.py    License: Apache License 2.0 5 votes vote down vote up
def ReportEsts(estimators):
    v = -1
    for est in estimators:
        print(est.name, 'max', np.max(est.errs), '99th',
              np.quantile(est.errs, 0.99), '95th', np.quantile(est.errs, 0.95),
              'median', np.quantile(est.errs, 0.5))
        v = max(v, np.max(est.errs))
    return v 
Example 14
Project: whynot   Author: zykls   File: experiments.py    License: MIT License 5 votes vote down vote up
def mediation_propensity_scores(intervention, untreated_runs):
    """Probability of treating each unit.

    Units with the largest populations are more likely to be treated.
    """
    populations = [run[intervention.time].population for run in untreated_runs]
    upper_quantile = np.quantile(populations, 0.9)
    propensities = 0.05 * np.ones(len(untreated_runs))
    propensities[populations > upper_quantile] = 0.9
    return propensities 
Example 15
Project: whynot   Author: zykls   File: framework.py    License: MIT License 5 votes vote down vote up
def bootstrap_sample_ate_ci(self, num_bootstrap_samples=2000, alpha=0.05):
        """Bootstrap a (1-alpha)% confidence interval for the sample ate."""
        means = []
        for _ in range(num_bootstrap_samples):
            sample = np.random.choice(
                self.true_effects, size=len(self.true_effects), replace=True
            )
            means.append(np.mean(sample))
        lower_tail, upper_tail = alpha / 2.0, 1.0 - alpha / 2.0
        return (np.quantile(means, lower_tail), np.quantile(means, upper_tail)) 
Example 16
Project: recruit   Author: Frank-qlu   File: nanfunctions.py    License: Apache License 2.0 5 votes vote down vote up
def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 
Example 17
Project: recruit   Author: Frank-qlu   File: test_function_base.py    License: Apache License 2.0 5 votes vote down vote up
def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.quantile(x, 0), 0.)
        assert_equal(np.quantile(x, 1), 3.5)
        assert_equal(np.quantile(x, 0.5), 1.75) 
Example 18
Project: recruit   Author: Frank-qlu   File: test_function_base.py    License: Apache License 2.0 5 votes vote down vote up
def test_no_p_overwrite(self):
        # this is worth retesting, because quantile does not make a copy
        p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
        p = p0.copy()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0)

        p0 = p0.tolist()
        p = p.tolist()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0) 
Example 19
Project: VTuber_Unity   Author: kwea123   File: misc.py    License: MIT License 5 votes vote down vote up
def detect_iris(frame, marks, side='left'):
    """
    return:
       x: the x coordinate of the iris.
       y: the y coordinate of the iris.
       x_rate: how much the iris is toward the left. 0 means totally left and 1 is totally right.
       y_rate: how much the iris is toward the top. 0 means totally top and 1 is totally bottom.
    """
    mask = np.full(frame.shape[:2], 255, np.uint8)
    if side == 'left':
        region = marks[36:42].astype(np.int32)
    elif side == 'right':
        region = marks[42:48].astype(np.int32)
    try:
        cv2.fillPoly(mask, [region], (0, 0, 0))
        eye = cv2.bitwise_not(frame, frame.copy(), mask=mask)
        # Cropping on the eye
        margin = 4
        min_x = np.min(region[:, 0]) - margin
        max_x = np.max(region[:, 0]) + margin
        min_y = np.min(region[:, 1]) - margin
        max_y = np.max(region[:, 1]) + margin

        eye = eye[min_y:max_y, min_x:max_x]
        eye = cv2.cvtColor(eye, cv2.COLOR_RGB2GRAY)

        eye_binarized = cv2.threshold(eye, np.quantile(eye, 0.2), 255, cv2.THRESH_BINARY)[1]
        contours, _ = cv2.findContours(eye_binarized, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
        contours = sorted(contours, key=cv2.contourArea)
        moments = cv2.moments(contours[-2])
        x = int(moments['m10'] / moments['m00']) + min_x
        y = int(moments['m01'] / moments['m00']) + min_y
        return x, y, (x-min_x-margin)/(max_x-min_x-2*margin), (y-min_y-margin)/(max_y-min_y-2*margin)
    except:
        return 0, 0, 0.5, 0.5 
Example 20
Project: stdpopsim   Author: popsim-consortium   File: validation.py    License: GNU General Public License v3.0 5 votes vote down vote up
def tmrca(ts):
    """
    Time to most recent common ancestor of sample, aka tree height.
    """
    tmrcas = [tree.time(tree.root) for tree in ts.trees()]
    min_, median, max_ = np.quantile(tmrcas, (0, 0.5, 1))
    return {"min(tmrca)": min_,
            "median(tmrca)": median,
            "max(tmrca)": max_,
            } 
Example 21
Project: mars   Author: mars-project   File: quantile.py    License: Apache License 2.0 5 votes vote down vote up
def execute(cls, ctx, op):
        inputs, device_id, xp = as_same_device(
            [ctx[inp.key] for inp in op.inputs], device=op.device, ret_extra=True)
        a = inputs[0]
        out = inputs[-1].copy() if op.out is not None else None

        with device(device_id):
            ctx[op.outputs[0].key] = xp.quantile(a, q=op.q, axis=op.axis, out=out,
                                                 interpolation=op.interpolation,
                                                 keepdims=op.keepdims) 
Example 22
Project: lambda-packs   Author: ryfeus   File: nanfunctions.py    License: MIT License 5 votes vote down vote up
def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 
Example 23
Project: nevergrad   Author: facebookresearch   File: utils.py    License: MIT License 5 votes vote down vote up
def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]:
        if len(archive) < self.max_len:
            return archive
        quantiles: tp.Dict[str, float] = {}
        threshold = float(self.min_len) / len(archive)
        names = ["optimistic", "pessimistic", "average"]
        for name in names:
            quantiles[name] = np.quantile([v.get_estimation(name) for v in archive.values()], threshold, interpolation="lower")
        new_archive: Archive[MultiValue] = Archive()
        new_archive.bytesdict = {b: v for b, v in archive.bytesdict.items() if any(v.get_estimation(n) <= quantiles[n] for n in names)}
        return new_archive 
Example 24
Project: armory   Author: twosixlabs   File: spectral_signature_defense.py    License: MIT License 5 votes vote down vote up
def detect_poison(self, **kwargs):
        """
        Returns poison detected and a report.
        :return: (report, is_clean_lst):
                where a report is None (for future ART compatibility)
                where is_clean is a list, where is_clean_lst[i]=1 means that x_train[i]
                there is clean and is_clean_lst[i]=0, means that x_train[i] was classified as poison.
        """

        self.set_params(**kwargs)

        n_classes = self.classifier.nb_classes()
        nb_layers = len(self.classifier.layer_names)

        features_x_poisoned = self.classifier.get_activations(
            self.x_train, layer=nb_layers - 1, batch_size=self.batch_size
        )

        features_split = SpectralSignatureDefense.split_by_class(
            features_x_poisoned, self.y_train, n_classes
        )
        keep_by_class = []
        for idx, feature in enumerate(features_split):
            score = SpectralSignatureDefense.spectral_signature_scores(feature)
            score_cutoff = np.quantile(
                score, max(1 - self.eps_multiplier * self.ub_pct_poison, 0.0)
            )
            keep_by_class.append(score < score_cutoff)

        base_indices_by_class = SpectralSignatureDefense.split_by_class(
            np.arange(self.y_train.shape[0]), self.y_train, 10
        )
        is_clean_lst = np.zeros_like(self.y_train, dtype=np.int)

        for keep_booleans, indices in zip(keep_by_class, base_indices_by_class):
            for keep_boolean, idx in zip(keep_booleans, indices):
                if keep_boolean:
                    is_clean_lst[idx] = 1

        return None, is_clean_lst 
Example 25
Project: Mastering-Elasticsearch-7.0   Author: PacktPublishing   File: nanfunctions.py    License: MIT License 5 votes vote down vote up
def _nanquantile_1d(arr1d, q, overwrite_input=False, interpolation='linear'):
    """
    Private function for rank 1 arrays. Compute quantile ignoring NaNs.
    See nanpercentile for parameter usage
    """
    arr1d, overwrite_input = _remove_nan_1d(arr1d,
        overwrite_input=overwrite_input)
    if arr1d.size == 0:
        return np.full(q.shape, np.nan)[()]  # convert to scalar

    return function_base._quantile_unchecked(
        arr1d, q, overwrite_input=overwrite_input, interpolation=interpolation) 
Example 26
Project: Mastering-Elasticsearch-7.0   Author: PacktPublishing   File: test_function_base.py    License: MIT License 5 votes vote down vote up
def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.quantile(x, 0), 0.)
        assert_equal(np.quantile(x, 1), 3.5)
        assert_equal(np.quantile(x, 0.5), 1.75) 
Example 27
Project: Mastering-Elasticsearch-7.0   Author: PacktPublishing   File: test_function_base.py    License: MIT License 5 votes vote down vote up
def test_no_p_overwrite(self):
        # this is worth retesting, because quantile does not make a copy
        p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
        p = p0.copy()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0)

        p0 = p0.tolist()
        p = p.tolist()
        np.quantile(np.arange(100.), p, interpolation="midpoint")
        assert_array_equal(p, p0) 
Example 28
Project: driverlessai-recipes   Author: h2oai   File: marketing_campaign.py    License: Apache License 2.0 5 votes vote down vote up
def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None,
              **kwargs) -> float:
        # label actuals as 1 or 0
        lb = LabelEncoder()
        labels = lb.fit_transform(labels)
        actual = lb.transform(actual)

        # probability of predicted response likelihood above which we'll send a letter
        cutoff = np.quantile(predicted, self._quantile)
        # print("cutoff: %f" % cutoff)

        # whom we'll send letter to
        selected = (predicted >= cutoff).ravel()
        num_letters = np.count_nonzero(selected)
        # print("number of letters: %d" % num_letters)

        # compute cost and reward
        cost = num_letters * self._cost  # each letter costs _cost
        reward = np.count_nonzero(actual[selected] == 1) * self._reward  # each true positive leads to _reward
        # print("cost: %f" % cost)
        # print("reward: %f" % reward)

        # compute total net income
        net_income = reward - cost
        # print("net_income: %f" % net_income)

        # return mean profit per letter sent
        return 0 if num_letters == 0 else net_income / num_letters 
Example 29
Project: driverlessai-recipes   Author: h2oai   File: top_decile.py    License: Apache License 2.0 5 votes vote down vote up
def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None,
              **kwargs) -> float:
        cutoff = np.quantile(predicted, 0.9)
        which = (predicted >= cutoff).ravel()
        return float(np.median(np.abs(actual[which] - predicted[which]))) 
Example 30
Project: driverlessai-recipes   Author: h2oai   File: quantile_winsorizer.py    License: Apache License 2.0 5 votes vote down vote up
def get_parameter_choices():
        return {"quantile": [0.01, 0.001, 0.05]}