Python numpy.datetime64() Examples

The following are 30 code examples of numpy.datetime64(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module numpy , or try the search function .
Example #1
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_split(self):
        sampling = numpy.timedelta64(5, 's')
        points = 100000
        ts = carbonara.TimeSerie.from_data(
            timestamps=list(map(datetime.datetime.utcfromtimestamp,
                                six.moves.range(points))),
            values=list(six.moves.range(points)))
        agg = self._resample(ts, sampling, 'mean')

        grouped_points = list(agg.split())

        self.assertEqual(
            math.ceil((points / sampling.astype(float))
                      / carbonara.SplitKey.POINTS_PER_SPLIT),
            len(grouped_points))
        self.assertEqual("0.0",
                         str(carbonara.SplitKey(grouped_points[0][0], 0)))
        # 3600 × 5s = 5 hours
        self.assertEqual(datetime64(1970, 1, 1, 5),
                         grouped_points[1][0])
        self.assertEqual(carbonara.SplitKey.POINTS_PER_SPLIT,
                         len(grouped_points[0][1])) 
Example #2
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.assertRaises(storage.AggregationDoesNotExist,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, 'last'),
                           processor.MetricReference(metric2, 'last')],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "last"],
                              [(metric2.id), "last"],
                          ]]) 
Example #3
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_aggregated_different_archive_no_overlap2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = carbonara.AggregatedTimeSerie(
            carbonara.Aggregation('mean', numpy.timedelta64(60, 's'), None))

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        metric = mock.Mock(id=str(uuid.uuid4()))
        ref = processor.MetricReference(metric, "mean")
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], (ref, tsc2)],
                          operations=["aggregate", "mean",
                                      ["metric", tsc1['return'][0].lookup_key,
                                       ref.lookup_key]]) 
Example #4
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc2))

        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          operations=["aggregate", "mean", [
                              "metric", ["all", "mean"]]]) 
Example #5
Source File: carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def __getitem__(self, key):
        if isinstance(key, numpy.datetime64):
            idx = numpy.searchsorted(self.timestamps, key)
            if self.timestamps[idx] == key:
                return self[idx]
            raise KeyError(key)
        if isinstance(key, slice):
            if isinstance(key.start, numpy.datetime64):
                start = numpy.searchsorted(self.timestamps, key.start)
            else:
                start = key.start
            if isinstance(key.stop, numpy.datetime64):
                stop = numpy.searchsorted(self.timestamps, key.stop)
            else:
                stop = key.stop
            key = slice(start, stop, key.step)
        return self.ts[key] 
Example #6
Source File: times.py    From aospy with Apache License 2.0 6 votes vote down vote up
def extract_months(time, months):
    """Extract times within specified months of the year.

    Parameters
    ----------
    time : xarray.DataArray
         Array of times that can be represented by numpy.datetime64 objects
         (i.e. the year is between 1678 and 2262).
    months : Desired months of the year to include

    Returns
    -------
    xarray.DataArray of the desired times
    """
    inds = _month_conditional(time, months)
    return time.sel(time=inds) 
Example #7
Source File: times.py    From aospy with Apache License 2.0 6 votes vote down vote up
def ensure_datetime(obj):
    """Return the object if it is a datetime-like object

    Parameters
    ----------
    obj : Object to be tested.

    Returns
    -------
    The original object if it is a datetime-like object

    Raises
    ------
    TypeError if `obj` is not datetime-like
    """
    _VALID_TYPES = (str, datetime.datetime, cftime.datetime,
                    np.datetime64)
    if isinstance(obj, _VALID_TYPES):
        return obj
    raise TypeError("datetime-like object required.  "
                    "Type given: {}".format(type(obj))) 
Example #8
Source File: carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def truncate(self, oldest_point=None):
        """Truncate the time series up to oldest_point excluded.

        :param oldest_point: Oldest point to keep from, this excluded.
                             Default is the aggregation timespan.
        :type oldest_point: numpy.datetime64 or numpy.timedelta64
        :return: The oldest point that could have been kept.
        """
        last = self.last
        if last is None:
            return
        if oldest_point is None:
            oldest_point = self.aggregation.timespan
            if oldest_point is None:
                return
        if isinstance(oldest_point, numpy.timedelta64):
            oldest_point = last - oldest_point
        index = numpy.searchsorted(self.ts['timestamps'], oldest_point,
                                   side='right')
        self.ts = self.ts[index:]
        return oldest_point 
Example #9
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_corrupted_split(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
        ])
        self.trigger_processing()

        aggregation = self.metric.archive_policy.get_aggregation(
            "mean", numpy.timedelta64(5, 'm'))

        with mock.patch('gnocchi.carbonara.AggregatedTimeSerie.unserialize',
                        side_effect=carbonara.InvalidData()):
            results = self.storage._get_splits_and_unserialize({
                self.metric: {
                    aggregation: [
                        carbonara.SplitKey(
                            numpy.datetime64(1387800000, 's'),
                            numpy.timedelta64(5, 'm'))
                    ],
                },
            })[self.metric][aggregation]
            self.assertEqual(1, len(results))
            self.assertIsInstance(results[0], carbonara.AggregatedTimeSerie)
            # Assert it's an empty one since corrupted
            self.assertEqual(0, len(results[0]))
            self.assertEqual(results[0].aggregation, aggregation) 
Example #10
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_get_splits_and_unserialize(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
        ])
        self.trigger_processing()

        aggregation = self.metric.archive_policy.get_aggregation(
            "mean", numpy.timedelta64(5, 'm'))

        results = self.storage._get_splits_and_unserialize({
            self.metric: {
                aggregation: [
                    carbonara.SplitKey(
                        numpy.datetime64(1387800000, 's'),
                        numpy.timedelta64(5, 'm')),
                ],
            },
        })[self.metric][aggregation]
        self.assertEqual(1, len(results))
        self.assertIsInstance(results[0], carbonara.AggregatedTimeSerie)
        # Assert it's not empty one since corrupted
        self.assertGreater(len(results[0]), 0)
        self.assertEqual(results[0].aggregation, aggregation) 
Example #11
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_split_key(self):
        self.assertEqual(
            numpy.datetime64("2014-10-07"),
            carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01T15:03"),
                numpy.timedelta64(3600, 's')))
        self.assertEqual(
            numpy.datetime64("2014-12-31 18:00"),
            carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01 15:03:58"),
                numpy.timedelta64(58, 's')))

        key = carbonara.SplitKey.from_timestamp_and_sampling(
            numpy.datetime64("2015-01-01 15:03"),
            numpy.timedelta64(3600, 's'))

        self.assertGreater(key, numpy.datetime64("1970"))

        self.assertGreaterEqual(key, numpy.datetime64("1970")) 
Example #12
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_delete_nonempty_metric(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
        ])
        self.trigger_processing()
        self.storage._delete_metric(self.metric)
        self.trigger_processing()

        aggregations = (
            self.metric.archive_policy.get_aggregations_for_method("mean")
        )

        self.assertRaises(storage.MetricDoesNotExist,
                          self.storage.get_aggregated_measures,
                          {self.metric: aggregations})
        self.assertEqual(
            {self.metric: None},
            self.storage._get_or_create_unaggregated_timeseries(
                [self.metric])) 
Example #13
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_get_aggregated_measures(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, i, j), 100)
            for i in six.moves.range(0, 60) for j in six.moves.range(0, 60)])
        self.trigger_processing([self.metric])

        aggregations = self.metric.archive_policy.aggregations

        measures = self.storage.get_aggregated_measures(
            {self.metric: aggregations})
        self.assertEqual(1, len(measures))
        self.assertIn(self.metric, measures)
        measures = measures[self.metric]
        self.assertEqual(len(aggregations), len(measures))
        self.assertGreater(len(measures[aggregations[0]]), 0)
        for agg in aggregations:
            self.assertEqual(agg, measures[agg].aggregation) 
Example #14
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_no_truncation(self):
        ts = {'sampling': numpy.timedelta64(60, 's'), 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie()

        for i in six.moves.range(1, 11):
            tsb.set_values(numpy.array([
                (datetime64(2014, 1, 1, 12, i, i), float(i))],
                dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                before_truncate_callback=functools.partial(
                    self._resample_and_merge, agg_dict=ts))
            tsb.set_values(numpy.array([
                (datetime64(2014, 1, 1, 12, i, i + 1), float(i + 1))],
                dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                before_truncate_callback=functools.partial(
                    self._resample_and_merge, agg_dict=ts))
            self.assertEqual(i, len(list(ts['return'].fetch()))) 
Example #15
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_add_measures_update_subset(self):
        m, m_sql = self._create_metric('medium')
        measures = [
            incoming.Measure(datetime64(2014, 1, 6, i, j, 0), 100)
            for i in six.moves.range(2) for j in six.moves.range(0, 60, 2)]
        self.incoming.add_measures(m.id, measures)
        self.trigger_processing([m])

        # add measure to end, in same aggregate time as last point.
        new_point = datetime64(2014, 1, 6, 1, 58, 1)
        self.incoming.add_measures(m.id, [incoming.Measure(new_point, 100)])

        with mock.patch.object(self.incoming, 'add_measures') as c:
            self.trigger_processing([m])
        for __, args, __ in c.mock_calls:
            self.assertEqual(
                list(args[3])[0][0], carbonara.round_timestamp(
                    new_point, args[1].granularity * 10e8)) 
Example #16
Source File: test_storage.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_get_measure_unknown_aggregation(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])

        aggregations = (
            self.metric.archive_policy.get_aggregations_for_method("last")
        )

        self.assertRaises(
            storage.MetricDoesNotExist,
            self.storage.get_aggregated_measures,
            {self.metric: aggregations}) 
Example #17
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_aggregation_std_with_unique(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0)], [3])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'std')
        self.assertEqual(0, len(ts), ts.values)

        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9),
             datetime64(2014, 1, 1, 12, 1, 6)],
            [3, 6, 5, 9])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), "std")

        self.assertEqual(1, len(ts))
        self.assertEqual(1.5275252316519465,
                         ts[datetime64(2014, 1, 1, 12, 0, 0)][1]) 
Example #18
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_duplicate_timestamps(self):
        ts = carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [10, 23])
        self.assertEqual(2, len(ts))
        self.assertEqual(10.0, ts[0][1])
        self.assertEqual(23.0, ts[1][1])

        ts.set_values(numpy.array([(datetime64(2014, 1, 1, 13, 0, 10), 3),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 9),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 8),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 7),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 4)],
                                  dtype=carbonara.TIMESERIES_ARRAY_DTYPE))
        self.assertEqual(4, len(ts))
        self.assertEqual(10.0, ts[0][1])
        self.assertEqual(23.0, ts[1][1])
        self.assertEqual(3.0, ts[2][1])
        self.assertEqual(9.0, ts[3][1]) 
Example #19
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 6 votes vote down vote up
def test_derived_hole(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime.datetime(2014, 1, 1, 12, 0, 0),
             datetime.datetime(2014, 1, 1, 12, 0, 4),
             datetime.datetime(2014, 1, 1, 12, 1, 2),
             datetime.datetime(2014, 1, 1, 12, 1, 14),
             datetime.datetime(2014, 1, 1, 12, 1, 24),
             datetime.datetime(2014, 1, 1, 12, 3, 2),
             datetime.datetime(2014, 1, 1, 12, 3, 22),
             datetime.datetime(2014, 1, 1, 12, 3, 42),
             datetime.datetime(2014, 1, 1, 12, 4, 9)],
            [50, 55, 65, 66, 70, 105, 108, 200, 202])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'last',
                            derived=True)

        self.assertEqual(4, len(ts))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 0), 5),
             (datetime64(2014, 1, 1, 12, 1, 0), 4),
             (datetime64(2014, 1, 1, 12, 3, 0), 92),
             (datetime64(2014, 1, 1, 12, 4, 0), 2)],
            list(ts.fetch(
                from_timestamp=datetime64(2014, 1, 1, 12)))) 
Example #20
Source File: times.py    From aospy with Apache License 2.0 5 votes vote down vote up
def apply_time_offset(time, years=0, months=0, days=0, hours=0):
    """Apply a specified offset to the given time array.

    This is useful for GFDL model output of instantaneous values.  For example,
    3 hourly data postprocessed to netCDF files spanning 1 year each will
    actually have time values that are offset by 3 hours, such that the first
    value is for 1 Jan 03:00 and the last value is 1 Jan 00:00 of the
    subsequent year.  This causes problems in xarray, e.g. when trying to group
    by month.  It is resolved by manually subtracting off those three hours,
    such that the dates span from 1 Jan 00:00 to 31 Dec 21:00 as desired.

    Parameters
    ----------
    time : xarray.DataArray representing a timeseries
    years, months, days, hours : int, optional
        The number of years, months, days, and hours, respectively, to offset
        the time array by.  Positive values move the times later.

    Returns
    -------
    pandas.DatetimeIndex

    Examples
    --------
    Case of a length-1 input time array:

    >>> times = xr.DataArray(datetime.datetime(1899, 12, 31, 21))
    >>> apply_time_offset(times)
    Timestamp('1900-01-01 00:00:00')

    Case of input time array with length greater than one:

    >>> times = xr.DataArray([datetime.datetime(1899, 12, 31, 21),
    ...                       datetime.datetime(1899, 1, 31, 21)])
    >>> apply_time_offset(times) # doctest: +NORMALIZE_WHITESPACE
    DatetimeIndex(['1900-01-01', '1899-02-01'], dtype='datetime64[ns]',
                  freq=None)
    """
    return (pd.to_datetime(time.values) +
            pd.DateOffset(years=years, months=months, days=days, hours=hours)) 
Example #21
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_truncate(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(1, 's'), 'mean')

        ts.truncate(datetime64(2014, 1, 1, 12, 0, 0))

        self.assertEqual(2, len(ts))
        self.assertEqual(5, ts[0][1])
        self.assertEqual(6, ts[1][1]) 
Example #22
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_74_percentile_serialized(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '74pct')

        self.assertEqual(1, len(ts))
        self.assertEqual(5.48, ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

        # Serialize and unserialize
        key = ts.get_split_key()
        o, s = ts.serialize(key)
        saved_ts = carbonara.AggregatedTimeSerie.unserialize(
            s, key, ts.aggregation)

        self.assertEqual(ts.aggregation, saved_ts.aggregation)

        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '74pct')
        saved_ts.merge(ts)

        self.assertEqual(1, len(ts))
        self.assertEqual(5.48, ts[datetime64(2014, 1, 1, 12, 0, 0)][1]) 
Example #23
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["*", ["metric", str(self.metric.id), "mean"],
                  ["metric", str(metric2.id), "mean"]],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 621),
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 84),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 16),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 176),
        ], values) 
Example #24
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_95_percentile(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '95pct')

        self.assertEqual(1, len(ts))
        self.assertEqual(5.9000000000000004,
                         ts[datetime64(2014, 1, 1, 12, 0, 0)][1]) 
Example #25
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_down_sampling(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 7])
        ts = self._resample(ts, numpy.timedelta64(300, 's'), 'mean')

        self.assertEqual(1, len(ts))
        self.assertEqual(5, ts[datetime64(2014, 1, 1, 12, 0, 0)][1]) 
Example #26
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["-",
             2,
             ["resample", "mean", numpy.timedelta64(1, 'D'),
              ["metric",
               [str(self.metric.id), "mean"],
               [str(metric2.id), "mean"]]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -37.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -2.75)]
            }
        }, values) 
Example #27
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_down_sampling_and_truncate(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 1, 4),
             datetime64(2014, 1, 1, 12, 1, 9),
             datetime64(2014, 1, 1, 12, 2, 12)],
            [3, 5, 7, 1])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'mean')

        ts.truncate(datetime64(2014, 1, 1, 12, 0, 59))

        self.assertEqual(2, len(ts))
        self.assertEqual(6, ts[datetime64(2014, 1, 1, 12, 1, 0)][1])
        self.assertEqual(1, ts[datetime64(2014, 1, 1, 12, 2, 0)][1]) 
Example #28
Source File: test_carbonara.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_fetch_nano(self):
        ts = {'sampling': numpy.timedelta64(200, 'ms'),
              'size': 10, 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 0, 200123), 4),
            (datetime64(2014, 1, 1, 11, 46, 0, 340000), 8),
            (datetime64(2014, 1, 1, 11, 47, 0, 323154), 50),
            (datetime64(2014, 1, 1, 11, 48, 0, 590903), 4),
            (datetime64(2014, 1, 1, 11, 48, 0, 903291), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 48, 0, 821312), 5)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 11, 46, 0, 200000), 6.0),
            (datetime64(2014, 1, 1, 11, 47, 0, 200000), 50.0),
            (datetime64(2014, 1, 1, 11, 48, 0, 400000), 4.0),
            (datetime64(2014, 1, 1, 11, 48, 0, 800000), 4.5)
        ], list(ts['return'].fetch()))
        self.assertEqual(numpy.timedelta64(200000000, 'ns'),
                         ts['return'].aggregation.granularity) 
Example #29
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 31), 8),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 42),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 6),
            incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10), 2),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, 'mean'),
             processor.MetricReference(metric2, 'mean')],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 11.0),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 22.0)
        ], values) 
Example #30
Source File: test_aggregates.py    From gnocchi with Apache License 2.0 5 votes vote down vote up
def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["resample", "mean", numpy.timedelta64(1, 'D'),
             ["metric",
              [str(self.metric.id), "mean"],
              [str(metric2.id), "mean"]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 39.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 4.75)]
            }
        }, values)