# Copyright (c) 2019 NASK. All rights reserved.

import datetime
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

from n6lib.datetime_helpers import parse_iso_datetime_to_utc

class RequestPerformer(object):

    A simple yet flexible tool to download data with HTTP/HTTPS (able to deal
    with huge downloads in a memory-efficient manner).

    RequestPerformer is intended to be used in one of the following ways:

    (1) using its `fetch()` class method to perform a "one-shot" download (in
        this case the `stream` option is automatically forced to be `False`,
        so it cannot be passed in):

        content = RequestPerformer.fetch('GET', 'https://example.com')

    (2) using its *context manager* and *iterator* interfaces to download
        the content in a stream-like way (making it possible to deal with
        large amounts of data in a memory-efficient manner):

        with RequestPerformer('GET', 'https://example.com') as perf:
            for chunk in perf:  # `chunk` is a byte string (str)

    (3) using the *context manager* interface to initialize the download and
        use `requests.Response` and/or `requests.Session` objects directly
        (within its `with` block a `RequestPerformer` instance has the
        following public attributes: `request` set to `requests.Response`
        instance, and `session` set to `requests.Session` instance):

        with RequestPerformer('GET', 'https://example.com') as perf:
            print perf.response.content   # download all data now!
            print perf.response.headers   # get headers
            print perf.session.cookies    # get cookies (as a CookieJar instance)
            # etc...

    The (2) and (3) ways can be combined, except that -- if the `stream`
    keyword argument is true (see below) -- you can **either** use the
    *iterator* interface [see above: (2)] **or** get directly the `content`
    attribute of the `response` attribute [see above: (3)], but should
    **not** do both.

    Apart from specifying the HTTP method and the URL, you can also pass in a
    lot of other arguments (see the constructor args/kwargs described below),
    especially various options related to the `requests` library's stuff
    (used under the hood). For example:

        with RequestPerformer('POST',
                              data={'example': 'data'},
                              headers={'header-test: 'true'},
                              allow_redirects=True) as perf:
            # (...)

    Required constructor args/kwargs:

        `method` (str or unicode):
            The HTTP method name (the `SUPPORTED_HTTP_METHODS` class attribute
            contains its allowed values). To be passed into

        `url` (str or unicode):
            The URL to download from (must start with one of the strings
            the `SUPPORTED_URL_PREFIXES` class attribute contains). To be
            passed into `requests.Session.request()`.

    Optional constructor kwargs:

        `data` (str or file-like, or dict, or list of 2-tuples;
                default: `None`):
            The data to be optionally sent as the request body. To be
            passed into `requests.Session.request()`.

        `headers` (dict; default: `None`):
            The headers to be optionally attached to the request. To be passed
            into `requests.Session.request()`.

        `allow_redirects` (boolean; default: `False`):
            Set to `True` to allow redirects in case of
            To be passed into `requests.Session.request()`.

        `timeout` (int or float, or 2-tuple of ints/floats;
                   default: `(12.1, 25)`):
            The request timeout specification -- to be passed into
            `requests.Session.request()`. See also:

        `retries` (int; default: `0`):
            Maximum number of attempts to retry on request failures (such as
            a connection problem or a server-side error whose HTTP status code
            suggests that a repeated request may succeed). If non-zero, it is,
            along with `backoff_factor`, passed into the `Retry` constructor
            (as the `total` argument -- see: `urllib3.util.retry.Retry`).
            If equal to 0 (the default), no retries are attempted (that is,
            an exception is raised on first failure). Note that the `retries`
            number does not include the first request, e.g., `retries=3` means
            that there can be up to 4 request attempts (the first one plus
            maximum of 3 retries). *Beware* that, depending on the server
            semantics, each attempt may change the server-side state/data
            (especially when `method` is POST or PATCH).

        `backoff_factor` (float; default: `0.1`):
            Backoff factor to apply between retry attempts. If the `retries`
            argument is non-zero, the `backoff_factor` argument is passed into
            the `Retry` constructor (see: `urllib3.util.retry.Retry`).
            For non-zero `backoff_factor`, delays between requests increase.
            According to `urllib3` docs, the delay before the first retry is
            `0s` (i.e., the first retry is immediate), and the formula for
            consecutive delays is:
            `backoff_factor * (2 ** (<retry number> - 1)) seconds`
            (but *never more* than `urllib3.util.retry.Retry.BACKOFF_MAX`,
            that is, 120 seconds). Therefore, for `backoff_factor=0.1`
            (default) the sequence of delays between consecutive requests is:
            0s, 0.2s, 0.4s, 0.8s, 1.6s...

        `stream` (boolean; default: `True`):
            * If `True`: contents will be downloaded *in chunks* (see: the
              `chunk_size` argument described below) -- and *only* when:
              (a) the RequestPerformer instance is iterated over (downloading
                  one chunk per iteration step), or
              (b) the `response.content` attribute (or a similar attribute,
                  such as `response.text`...) is accessed the first time (then
                  all contents will be downloaded immediately and set as the
                  `response.content` attribute) -- however, note that if you
                  do not need the *stream-like* (iteration-based) way of
                  downloading the contents you probably just need to use the
                  `stream=False` variant (see below);
              note: the (a) and (b) ways should *not* be combined.
            * If `False`: contents will be downloaded *all at once*, just when
              entering the `with` block; then the contents will be accessible
              as the `response.content` attribute (and also by iteration over
              the RequestPerformer instance).
            This argument will also be passed into `requests.Session.request()`.

        `chunk_size` (int; default: `2 ** 16`):
            When downloading data in the stream-like way -- the amount
            of data (in bytes) to be transferred per iteration. Relevant
            only if the `stream` argument is true, otherwise ignored.
            Note: it is not necessarily the length of each yielded data
            chunk (because of decoding...).

        `custom_session_attrs` (dict; default: `None`):
            Custom `requests.Session()` instance attribute values.
            http://docs.python-requests.org/en/master/user/advanced/ and

        Other arbitrary kwargs:
            To be passed into `requests.Session.request()`.

    Exceptions raised by the constructor and/or methods:
        * ValueError -- for:
            * unsupported `method`/`url` values;
            * `data` being a file-like object whose content's length
              cannot be determined without consuming it, *and*, at the
              same time, non-zero `retries` given.
        * Any exception that can be raised by the `requests` or `urrlib3`

    For more information about args/kwargs related to the `requests` or
    `urllib3` libraries mentioned above -- see:
        * http://docs.python-requests.org/en/stable/api/
        * http://docs.python-requests.org/en/stable/user/advanced/
        * https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.retry.Retry

    SUPPORTED_URL_PREFIXES = ('http://', 'https://')

        'redirect': False,
        'respect_retry_after_header': False,
        # ad `status_forcelist` -- see: https://tools.ietf.org/html/rfc7231#section-6.6
        'status_forcelist': {500, 502, 503, 504},
        'method_whitelist': False,

    # sequence of datetime.strptime() formats tried (consecutively) by
    # the get_dt_header() method to parse date+time response headers
    # (see also: https://tools.ietf.org/html/rfc7231#section-
        # *note:* these 3 formats require that the C locale is set
        # (`n6lib` already ensures that; see n6lib/__init__.py...)

        # the preferred format
        '%a, %d %b %Y %H:%M:%S GMT',

        # old RFC-850 format
        '%A, %d-%b-%y %H:%M:%S GMT',

        # old ANSI C's asctime() format
        # (note: using '%d' here is OK, because datetime.strptime()
        # is lenient about '%d' vs. numbers that are *not* zero-padded,
        # as well as about extra spaces *between* input string elements).
        '%a %b %d %H:%M:%S %Y',

        # (apart from trying the above 3 formats, the get_dt_header()
        # method tries -- as the last attempt -- ISO 8601 parsing)

    def __init__(self,
                 timeout=(12.1, 25),
                 chunk_size=(2 ** 16),

        method = self._get_valid_method(method=method)
        url = self._get_valid_url(url=url)
        self.session = None             # to be set in __enter__()
        self.response = None            # to be set in __enter__()
        self._actual_iterator = None    # to be set in __enter__()
        self._custom_session_attrs = custom_session_attrs
        self._request_kwargs = dict(extra_request_kwargs,
        self._retry_conf = self._get_retry_conf(retries=retries,
        self._chunk_size = chunk_size if stream else None

    def fetch(cls, *args, **kwargs):
        Download all content at once.

            The same as constructor args/kwargs, except that `stream`
            is forbidden (internally it is forcibly set to `False`).

            The downloaded content (str).

            * ValueError: for unsupported `method`/`url` values.
            * Any exception that can be raised by the `requests` or `urrlib3`
        with RequestPerformer(*args, stream=False, **kwargs) as perf:
            return perf.response.content

    def __enter__(self):
        self.session = requests.Session()
            self.response = self.session.request(**self._request_kwargs)
            self._actual_iterator = self.response.iter_content(chunk_size=self._chunk_size)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):

    def __iter__(self):
        return self

    def next(self):
        return next(self._actual_iterator)

    def get_dt_header(self, header_key):
        A helper method to retrieve a response header as a date+time.

                The name of the HTTP response header.

            `None` or UTC date+time as a `datetime.datetime` instance
            (a naive one, i.e., without explicit timezone information).

        Example usage:
            with RequestPerformer('GET', 'http://example.com/FOO') as perf:
                foo_last_modified = perf.get_dt_header('Last-Modified')
            if foo_last_modified is None:
                print 'I have no idea when FOO was modified.`
                print 'FOO modification date+time:', foo_last_modified.isoformat()
        raw_value = (self.response.headers.get(header_key) or '').strip()
        if raw_value:
            for dt_format in self._HTTP_DATETIME_FORMATS:
                    return datetime.datetime.strptime(raw_value, dt_format)
                except ValueError:
                return parse_iso_datetime_to_utc(raw_value)
            except ValueError:
        return None

    def _get_valid_method(self, method):
        method = method.upper()
        if method not in self.SUPPORTED_HTTP_METHODS:
            raise ValueError('HTTP method {!r} not supported'.format(method))
        return method

    def _get_valid_url(self, url):
        url = self._get_url_with_lowercased_proto(url)
        if not url.startswith(self.SUPPORTED_URL_PREFIXES):
            raise ValueError('URL prefix {!r} not supported'.format(url))
        return url

    def _get_url_with_lowercased_proto(url):
        url_parts = url.split(':', 1)
        url_parts[0] = url_parts[0].lower()
        url = ':'.join(url_parts)
        return url

    def _get_retry_conf(self, retries, backoff_factor):
        if retries:
            retry_conf = Retry(backoff_factor=backoff_factor,
            return retry_conf
        return None

    def _set_custom_session_attrs(self):
        if self._custom_session_attrs:
            for name, value in self._custom_session_attrs.items():
                setattr(self.session, name, value)

    def _set_up_retries(self):
        if self._retry_conf is not None:
            for http_prefix in self.SUPPORTED_URL_PREFIXES:

class _HTTPAdapterForRetries(HTTPAdapter):

    def send(self, request, *args, **kwargs):
        content_length_is_unknown = (request.body is not None
                                     and 'Content-Length' not in request.headers)
        if content_length_is_unknown:
            # it seems that requests's HTTPAdapter does not perform
            # retries in such a case, even though they were requested
            # (see the source code of HTTPAdapter.send() in conjunction
            # with urllib3.connectionpool.HTTPConnectionPool.urlopen()
            # and urllib3.util.retry.Retry.increment()...) -- so here
            # we raise an exception to prevent such a silent omission
            # [we analyzed this for requests==2.21.0 and urllib3==1.24.1]
            raise ValueError('non-zero `retries` has been specified and, '
                             'at the same time, Content-Length of the request '
                             'could not be determined (suggested solutions: '
                             'specify `data` whose length is discoverable, '
                             'or specify `retries=0`)')

        return super(_HTTPAdapterForRetries, self).send(request, *args, **kwargs)