Python pycurl.FOLLOWLOCATION Examples

def _create_curl_conn(self, url):
        """
        Create a cURL connection object with useful default settings.
        """
        headers = []
        if self.user is not None and self.secret is not None:
            b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
            headers.append("Authorization: Basic {}".format(b64cred))

        conn = pycurl.Curl()

        if len(headers) > 0:
            conn.setopt(pycurl.HTTPHEADER, headers)

        conn.setopt(pycurl.URL, url)

        # github often redirects
        conn.setopt(pycurl.FOLLOWLOCATION, 1)

        return conn 

def _create_curl_conn(self, url):
        """
        Create a cURL connection object with useful default settings.
        """
        headers = []
        if self.user is not None and self.secret is not None:
            b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
            headers.append("Authorization: Basic {}".format(b64cred))

        conn = pycurl.Curl()

        if len(headers) > 0:
            conn.setopt(pycurl.HTTPHEADER, headers)

        conn.setopt(pycurl.URL, url)

        # github often redirects
        conn.setopt(pycurl.FOLLOWLOCATION, 1)

        return conn 

def test_pycurl_insecure(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com/get-ca-cert", curl=curl,
                       insecure=True)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com/get-ca-cert",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.SSL_VERIFYPEER: False,
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def test_headers(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com",
                       headers={"a": "1", "b": "2"}, curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.HTTPHEADER: ["a: 1", "b: 2"],
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def test_cainfo(self):
        curl = CurlStub(b"result")
        result = fetch("https://example.com", cainfo="cainfo", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"https://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.CAINFO: b"cainfo",
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def test_post_data(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", post=True, data="data", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options[pycurl.READFUNCTION](), b"data")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.POST: True,
                          pycurl.POSTFIELDSIZE: 4,
                          pycurl.READFUNCTION: Any(),
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def test_post(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", post=True, curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.POST: True,
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def generate_curl(self, url=None, headers=None):
        """ 生成一个curl, 返回 curl 实例和用于获取结果的 buffer
        """
        curl = pycurl.Curl()
        buff = StringIO()

        curl.setopt(pycurl.COOKIEFILE, "cookie")
        curl.setopt(pycurl.COOKIEJAR, "cookie_jar")
        curl.setopt(pycurl.SHARE, self.http._share)
        curl.setopt(pycurl.WRITEFUNCTION, buff.write)
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.TIMEOUT, 3)
        curl.setopt(pycurl.CONNECTTIMEOUT, 3)

        if url:
            curl.setopt(pycurl.URL, url)

        if headers:
            self.set_curl_headers(curl, headers)

        return curl, buff 

def url_check(self, url):
        '''下载地址检查'''

        url_info = {}
        proto = urlparse.urlparse(url)[0]
        if proto not in VALIDPROTOCOL:
            print 'Valid protocol should be http or ftp, but % s found < %s >!' % (proto, url)
        else:
            ss = StringIO()
            curl = pycurl.Curl()
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.setopt(pycurl.MAXREDIRS, 5)
            curl.setopt(pycurl.CONNECTTIMEOUT, 30)
            curl.setopt(pycurl.TIMEOUT, 300)
            curl.setopt(pycurl.NOSIGNAL, 1)
            curl.setopt(pycurl.NOPROGRESS, 1)
            curl.setopt(pycurl.NOBODY, 1)
            curl.setopt(pycurl.HEADERFUNCTION, ss.write)
            curl.setopt(pycurl.URL, url)

        try:
            curl.perform()
        except:
            pass

        if curl.errstr() == '' and curl.getinfo(pycurl.RESPONSE_CODE) in STATUS_OK:
            url_info['url'] = curl.getinfo(pycurl.EFFECTIVE_URL)
            url_info['file'] = os.path.split(url_info['url'])[1]
            url_info['size'] = int(
                curl.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
            url_info['partible'] = (ss.getvalue().find('Accept - Ranges') != -1)

        return url_info 

def test_request_get(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=200)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        method = 'get'
        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': ['bar2a', 'bar2b']}
        handlers = {200: self._handler_dummy}

        result = pyaem.bagofrequests.request(method, url, params, handlers)

        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2a&foo2=bar2b')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 4 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 4)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with()

        self.assertEqual(result.is_success(), True)
        self.assertEqual(result.message, 'some dummy message')
        self.assertEqual(result.response['request']['method'], 'get')
        self.assertEqual(result.response['request']['url'],
            'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2a&foo2=bar2b')
        self.assertEqual(result.response['request']['params'], params) 

def test_request_post(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=200)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        method = 'post'
        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': ['bar2a', 'bar2b']}
        handlers = {200: self._handler_dummy}

        result = pyaem.bagofrequests.request(method, url, params, handlers)

        curl.setopt.assert_any_call(pycurl.POST, 1)
        curl.setopt.assert_any_call(pycurl.POSTFIELDS, 'foo1=bar1&foo2=bar2a&foo2=bar2b')
        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 6 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 6)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with()

        self.assertEqual(result.is_success(), True)
        self.assertEqual(result.message, 'some dummy message')
        self.assertEqual(result.response['request']['method'], 'post')
        self.assertEqual(result.response['request']['url'], 'http://localhost:4502/.cqactions.html')
        self.assertEqual(result.response['request']['params'], params) 

def _follow_location_and_auto_referer(self):
        self._curl.setopt(pycurl.FOLLOWLOCATION, True)
        self._curl.setopt(pycurl.AUTOREFERER, True)
        self._curl.setopt(pycurl.NOBODY, True) 

def __init__(self, base_url="", fakeheaders=[]):
        self.handle = pycurl.Curl()
        # These members might be set.
        self.set_url(base_url)
        self.verbosity = 0
        self.fakeheaders = fakeheaders
        # Nothing past here should be modified by the caller.
        self.payload = None
        self.payload_io = BytesIO()
        self.hrd = ""
        # Verify that we've got the right site; harmless on a non-SSL connect.
        self.set_option(pycurl.SSL_VERIFYHOST, 2)
        # Follow redirects in case it wants to take us to a CGI...
        self.set_option(pycurl.FOLLOWLOCATION, 1)
        self.set_option(pycurl.MAXREDIRS, 5)
        self.set_option(pycurl.NOSIGNAL, 1)
        # Setting this option with even a nonexistent file makes libcurl
        # handle cookie capture and playback automatically.
        self.set_option(pycurl.COOKIEFILE, "/dev/null")
        # Set timeouts to avoid hanging too long
        self.set_timeout(30)
        # Use password identification from .netrc automatically
        self.set_option(pycurl.NETRC, 1)
        self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write)
        def header_callback(x):
            self.hdr += x.decode('ascii')
        self.set_option(pycurl.HEADERFUNCTION, header_callback) 

def test_request_head(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=200)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        method = 'head'
        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': ['bar2a', 'bar2b']}
        handlers = {200: self._handler_dummy}

        result = pyaem.bagofrequests.request(method, url, params, handlers)

        curl.setopt.assert_any_call(pycurl.HEADER, True)
        curl.setopt.assert_any_call(pycurl.NOBODY, True)
        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 6 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 6)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with()

        self.assertEqual(result.is_success(), True)
        self.assertEqual(result.message, 'some dummy message')
        self.assertEqual(result.response['request']['method'], 'head')
        self.assertEqual(result.response['request']['url'], 'http://localhost:4502/.cqactions.html')
        self.assertEqual(result.response['request']['params'], params) 

def get_url (url, user_agent=UA, referrer=None):
    """Make a GET request of the url using pycurl and return the data
    (which is None if unsuccessful)"""

    data = None
    databuffer = StringIO()

    curl = pycurl.Curl()
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 8)
    curl.setopt(pycurl.WRITEFUNCTION, databuffer.write)
    curl.setopt(pycurl.COOKIEFILE, '')
    if user_agent:
        curl.setopt(pycurl.USERAGENT, user_agent)
    if referrer is not None:
        curl.setopt(pycurl.REFERER, referrer)
    try:
        curl.perform()
        data = databuffer.getvalue()
    except Exception:
        pass
    curl.close()

    return data 

def test_request_unexpected_resp(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=500)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        method = 'get'
        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': 'bar2'}
        handlers = {}

        try:
            pyaem.bagofrequests.request(method, url, params, handlers)
            self.fail('An exception should have been raised')
        except pyaem.PyAemException as exception:
            self.assertEqual(exception.code, 500)
            self.assertEqual(exception.message, 'Unexpected response\nhttp code: 500\nbody:\n')

        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 4 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 4)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with() 

def test_download_file(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=200)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': ['bar2a', 'bar2b']}
        handlers = {200: self._handler_dummy}

        result = pyaem.bagofrequests.download_file(url, params, handlers, file='/tmp/somefile')

        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2a&foo2=bar2b')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 4 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 4)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with()

        self.assertEqual(result.is_success(), True)
        self.assertEqual(result.message, 'some dummy message')
        self.assertEqual(result.response['request']['method'], 'get')
        self.assertEqual(result.response['request']['url'],
            'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2a&foo2=bar2b')
        self.assertEqual(result.response['request']['params'], params) 

def test_download_file_unexpected(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=500)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': 'bar2'}
        handlers = {}

        try:
            pyaem.bagofrequests.download_file(url, params, handlers, file='/tmp/somefile')
            self.fail('An exception should have been raised')
        except pyaem.PyAemException as exception:
            self.assertEqual(exception.code, 500)
            self.assertEqual(exception.message,
                'Unexpected response\nhttp code: 500\nbody:\n' +
                'Download http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2 to /tmp/somefile')

        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 5 calls including the one with pycurl.WRITEDATA and pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 4)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with() 

def test_upload_file(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=200)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': 'bar2'}
        handlers = {200: self._handler_dummy}

        result = pyaem.bagofrequests.upload_file(url, params, handlers, file='/tmp/somefile')

        curl.setopt.assert_any_call(pycurl.POST, 1)
        curl.setopt.assert_any_call(pycurl.HTTPPOST, [('foo1', 'bar1'), ('foo2', 'bar2')])
        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 6 calls including the one with pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 6)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with()

        self.assertEqual(result.is_success(), True)
        self.assertEqual(result.message, 'some dummy message')
        self.assertEqual(result.response['request']['method'], 'post')
        self.assertEqual(result.response['request']['url'], 'http://localhost:4502/.cqactions.html')
        self.assertEqual(result.response['request']['params'], params) 

def download_url(host_id, url):
    """
    Downloads an URL and stores the response to a directory with named as the host/IP
    :param host_id:
    :param url:
    :return:
    """
    output = StringIO.StringIO()
    header = StringIO.StringIO()

    print "[>] Trying to download URL: %s" % url
    # Download file
    try:
        # 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)')
        c = pycurl.Curl()
        c.setopt(c.URL, url)
        c.setopt(pycurl.CONNECTTIMEOUT, 10)
        c.setopt(pycurl.TIMEOUT, 180)
        c.setopt(pycurl.FOLLOWLOCATION, 1)
        c.setopt(pycurl.USERAGENT, 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)')
        c.setopt(c.WRITEFUNCTION, output.write)
        c.setopt(c.HEADERFUNCTION, header.write)
        c.perform()
        # Header parsing
        header_info = header_function(header.getvalue())
    except Exception, e:
        if args.debug:
            traceback.print_exc()
        print_highlighted("[-] Error MESSAGE: %s" % str(e))

    # Write File 

def test_secure_download_no_cainfo(self):
        import pycurl
        import io

        # Required mocks
        mock_curl_instance = Mock()
        mock_stringio_instance = Mock()

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        expected_setopt_calls = [
            call(pycurl.URL, b"https://example.com/file.extension"),
            call(pycurl.SSL_VERIFYPEER, 1),
            call(pycurl.SSL_VERIFYHOST, 2),
            call(pycurl.WRITEFUNCTION, mock_stringio_instance.write),
            call(pycurl.FOLLOWLOCATION, 1),
            call(pycurl.MAXREDIRS, 10),
        ]

        with patch.object(pycurl, "Curl") as mock_curl_constructor:
            with patch.object(io, "StringIO") as mock_stringio_constructor:
                mock_curl_constructor.return_value = mock_curl_instance
                mock_stringio_constructor.return_value = mock_stringio_instance

                secure_download_result = helpers.secure_download(
                    "https://example.com/file.extension"
                )

        # Check curl calls
        mock_curl_instance.setopt.assert_has_calls(expected_setopt_calls)
        mock_curl_instance.perform.assert_called_once_with()
        mock_curl_instance.close.assert_called_once_with()

        # Check StringIO call
        mock_stringio_instance.getvalue.assert_called_once_with()

        self.assertEqual(secure_download_result, "mock StringIO value") 

def test_secure_download(self):
        import pycurl
        import io

        # Required mocks
        mock_curl_instance = Mock()
        mock_stringio_instance = Mock()

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        # Expected pycurl setopt calls
        expected_setopt_calls = [
            call(pycurl.URL, b"https://example.com/file.extension"),
            call(pycurl.SSL_VERIFYPEER, 1),
            call(pycurl.SSL_VERIFYHOST, 2),
            call(pycurl.CAINFO, "/etc/certs/cabundle.pem"),
            call(pycurl.WRITEFUNCTION, mock_stringio_instance.write),
            call(pycurl.FOLLOWLOCATION, 1),
            call(pycurl.MAXREDIRS, 10),
        ]

        with patch.object(pycurl, "Curl") as mock_curl_constructor:
            with patch.object(io, "StringIO") as mock_stringio_constructor:
                mock_curl_constructor.return_value = mock_curl_instance
                mock_stringio_constructor.return_value = mock_stringio_instance

                secure_download_result = helpers.secure_download(
                    "https://example.com/file.extension", "/etc/certs/cabundle.pem"
                )

        # Check curl calls
        mock_curl_instance.setopt.assert_has_calls(expected_setopt_calls)
        mock_curl_instance.perform.assert_called_once_with()
        mock_curl_instance.close.assert_called_once_with()

        # Check StringIO call
        mock_stringio_instance.getvalue.assert_called_once_with()

        self.assertEqual(secure_download_result, "mock StringIO value") 

def test_get_html_http_no_callbacks(self):
        import pycurl
        import io

        # Test http with no callbacks
        mock_curl_instance = Mock()
        mock_stringio_instance = Mock()

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        # Expected pycurl setopt calls
        expected_setopt_calls = [
            call(pycurl.URL, b"https://example.com/file.extension"),
            call(pycurl.WRITEFUNCTION, mock_stringio_instance.write),
            call(pycurl.FOLLOWLOCATION, 1),
            call(pycurl.MAXREDIRS, 10),
            call(
                pycurl.USERAGENT,
                "Fedora Upstream Release Monitoring "
                "(https://fedoraproject.org/wiki/Upstream_release_monitoring)",
            ),
            call(pycurl.CONNECTTIMEOUT, 10),
            call(pycurl.TIMEOUT, 30),
        ]

        with patch.object(pycurl, "Curl") as mock_curl_constructor:
            with patch.object(io, "StringIO") as mock_stringio_constructor:
                mock_curl_constructor.return_value = mock_curl_instance
                mock_stringio_constructor.return_value = mock_stringio_instance

                get_html_result = helpers.get_html("https://example.com/file.extension")

        # Check curl calls
        mock_curl_instance.setopt.assert_has_calls(expected_setopt_calls)
        mock_curl_instance.perform.assert_called_once_with()
        mock_curl_instance.close.assert_called_once_with()

        self.assertEqual(get_html_result, "mock StringIO value") 

def secure_download(url, cainfo=""):
    import pycurl
    import io

    c = pycurl.Curl()
    c.setopt(pycurl.URL, url.encode("ascii"))

    # -k / --insecure
    # c.setopt(pycurl.SSL_VERIFYPEER, 0)

    # Verify certificate
    c.setopt(pycurl.SSL_VERIFYPEER, 1)

    # Verify CommonName or Subject Alternate Name
    c.setopt(pycurl.SSL_VERIFYHOST, 2)

    # --cacert
    if cainfo:
        c.setopt(pycurl.CAINFO, cainfo)

    res = io.StringIO()

    c.setopt(pycurl.WRITEFUNCTION, res.write)

    # follow up to 10 http location: headers
    c.setopt(pycurl.FOLLOWLOCATION, 1)
    c.setopt(pycurl.MAXREDIRS, 10)

    c.perform()
    c.close()
    data = res.getvalue()
    res.close()

    return data 

def curl(self):
        """Sending a single cURL request to download"""
        c = self._pycurl
        # Resume download
        if os.path.exists(self.path) and self.resume:
            mode = 'ab'
            self.downloaded = os.path.getsize(self.path)
            c.setopt(pycurl.RESUME_FROM, self.downloaded)
        else:
            mode = 'wb'
        with open(self.path, mode) as f:
            c.setopt(c.URL, utf8_encode(self.url))
            if self.auth:
                c.setopt(c.USERPWD, '%s:%s' % self.auth)
            c.setopt(c.USERAGENT, self._user_agent)
            c.setopt(c.WRITEDATA, f)
            h = self._get_pycurl_headers()
            if h is not None:
                c.setopt(pycurl.HTTPHEADER, h)
            c.setopt(c.NOPROGRESS, 0)
            c.setopt(pycurl.FOLLOWLOCATION, 1)
            c.setopt(c.PROGRESSFUNCTION, self.progress)
            self._fill_in_cainfo()
            if self._pass_through_opts:
                for key, value in self._pass_through_opts.items():
                    c.setopt(key, value)
            c.perform() 

def test_create_curl(self):
        curls = []

        def pycurl_Curl():
            curl = CurlStub(b"result")
            curls.append(curl)
            return curl
        Curl = pycurl.Curl
        try:
            pycurl.Curl = pycurl_Curl
            result = fetch("http://example.com")
            curl = curls[0]
            self.assertEqual(result, b"result")
            self.assertEqual(curl.options,
                             {pycurl.URL: b"http://example.com",
                              pycurl.FOLLOWLOCATION: 1,
                              pycurl.MAXREDIRS: 5,
                              pycurl.CONNECTTIMEOUT: 30,
                              pycurl.LOW_SPEED_LIMIT: 1,
                              pycurl.LOW_SPEED_TIME: 600,
                              pycurl.NOSIGNAL: 1,
                              pycurl.WRITEFUNCTION: Any(),
                              pycurl.DNS_CACHE_TIMEOUT: 0,
                              pycurl.ENCODING: b"gzip,deflate"})
        finally:
            pycurl.Curl = Curl 

def test_pycurl_follow_false(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl,
                       follow=False)
        self.assertEqual(result, b"result")
        self.assertNotIn(pycurl.FOLLOWLOCATION, curl.options.keys()) 

def test_pycurl_follow_true(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl,
                       follow=True)
        self.assertEqual(result, b"result")
        self.assertEqual(1, curl.options[pycurl.FOLLOWLOCATION]) 

def test_basic(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com", curl=curl)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 

def request(self, endpoint, headers=None, post=None, first=True):
        buffer = BytesIO()

        ch = pycurl.Curl()

        ch.setopt(pycurl.URL, endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        if headers:
            ch.setopt(pycurl.HTTPHEADER, headers)

        ch.setopt(pycurl.VERBOSE, self.debug)
        ch.setopt(pycurl.SSL_VERIFYPEER, False)
        ch.setopt(pycurl.SSL_VERIFYHOST, False)
        ch.setopt(pycurl.COOKIEFILE, self.settingsPath + self.username + '-cookies.dat')
        ch.setopt(pycurl.COOKIEJAR, self.settingsPath + self.username + '-cookies.dat')

        if post:
            import urllib
            ch.setopt(pycurl.POST, len(post))
            ch.setopt(pycurl.POSTFIELDS, urllib.urlencode(post))

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]
        ch.close()

        if self.debug:
            import urllib
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print('DATA: ' + urllib.unquote_plus(json.dumps(post)))
            print("RESPONSE: " + body + "\n")

        return [header, json_decode(body)]