Python requests.head() Examples

The following are 30 code examples for showing how to use requests.head(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module requests , or try the search function .

Example 1
Project: python-wp   Author: myles   File: api.py    License: MIT License 6 votes vote down vote up
def _get_wp_api_url(self, url):
        """
        Private function for finding the WP-API URL.

        Arguments
        ---------

        url : str
            WordPress instance URL.
        """
        resp = requests.head(url)

        # Search the Links for rel="https://api.w.org/".
        wp_api_rel = resp.links.get('https://api.w.org/')

        if wp_api_rel:
            return wp_api_rel['url']
        else:
            # TODO: Rasie a better exception to the rel doesn't exist.
            raise Exception 
Example 2
Project: PythonEXE   Author: jabbalaci   File: hello.py    License: MIT License 6 votes vote down vote up
def main():
    print('hello world')
    print('contacting google.com...')
    r = requests.head("https://www.google.com")
    print("status code:", r.status_code)
    print("PATH:", os.getenv("PATH"))
    result = helper.add(1, 1)
    print(f"1 + 1 = {result}")
    try:
        with open(Path(BASE_DIR, "input.txt")) as f:
            content = f.read().strip()
        print(content)
    except IOError:
        print("Warning: input.txt was not found")
    #
    fname = Path(BASE_DIR, "output.txt")
    print("writing to the following file:", fname)
    with open(fname, "w") as g:
        print("writing to a file", file=g)

############################################################################## 
Example 3
Project: Awesome-Mobile-UI   Author: cymcsg   File: generate.py    License: Apache License 2.0 6 votes vote down vote up
def download_file(url, filename):
    # NOTE the stream=True parameter
    # response = requests.head(url)

    # if "Content-Length" in response.headers:
    #     print("length---" + response.headers["Content-Length"])
    # if "Content-Length" in response.headers and int(response.headers["Content-Length"]) > 100000:
    #     r = requests.get(url, stream=True)
    #     with open(filename, 'wb') as f:
    #         for chunk in r.iter_content(chunk_size=1024):
    #             if chunk:  # filter out keep-alive new chunks
    #                 f.write(chunk)
    #                 f.flush()
    # return filename
    r = requests.get(url, stream=True)
    if len(r.content) > 50000:
        # print("img url----" + url + "     " + str(len(r.content)))
        with open(filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
        return True
    else:
        return False 
Example 4
Project: anime-downloader   Author: vn-ki   File: selescrape.py    License: The Unlicense 6 votes vote down vote up
def status_select(driver, url, status='hide'):
    '''
    For now it doesnt do what its name suggests, 
    I have planned to add a status reporter of the http response code.
    This part of the code is not removed because it is part of its core.
    Treat it like it isnt here.
    '''
    try:
        if status == 'hide':
            driver.get(url)
        elif status == 'show':
            r = requests.head(url)
            if r.status_code == 503:
                raise RuntimeError("This website's sevice is unavailable or has cloudflare on.")
            driver.get(url)
            return r.status_code
        else:
            driver.get(url)
    except requests.ConnectionError:
        raise RuntimeError("Failed to establish a connection using the requests library.") 
Example 5
Project: orchestra   Author: google   File: display_video_360.py    License: Apache License 2.0 6 votes vote down vote up
def _download_report(source_url, destination_file, chunk_size):
        response = requests.head(source_url)
        content_length = int(response.headers['Content-Length'])

        start_byte = 0
        while start_byte < content_length:
            end_byte = start_byte + chunk_size - 1
            if end_byte >= content_length:
                end_byte = content_length - 1

            headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)}
            response = requests.get(source_url, stream=True, headers=headers)
            chunk = response.raw.read()
            destination_file.write(chunk)
            start_byte = end_byte + 1
        destination_file.close() 
Example 6
Project: CrawlBox   Author: abaykan   File: crawlbox.py    License: The Unlicense 6 votes vote down vote up
def checkUrl(url):

    # check
    try:
        ress1 = requests.head(url , allow_redirects=True)

        if url != ress1.url:
            return "Maybe you should use ;"+ress1.url
        else:
            ress = requests.get(url)
            code = ress.status_code
            if (code == 200):
                return True
            else:
               return False
    except requests.exceptions.ConnectionError:
        return "Try a different url please"
    except requests.exceptions.MissingSchema:
        return "Try a different url please"
    except:
        return False

## make input work both in python3 and 2 
Example 7
Project: bioconda-utils   Author: bioconda   File: bioconductor_skeleton.py    License: MIT License 6 votes vote down vote up
def cargoport_url(self):
        """
        Returns the Galaxy cargo-port URL for this version of this package, if
        it exists.
        """
        url = cargoport_url(self.package, self.version, self.bioc_version)
        response = requests.head(url)
        if response.status_code == 404:
            # This is expected if this is a new package or an updated version.
            # Cargo Port will archive a working URL upon merging
            return
        elif response.status_code == 200:
            return url
        else:
            raise PageNotFoundError(
                "Unexpected error: {0.status_code} ({0.reason})".format(response)) 
Example 8
Project: bioconda-utils   Author: bioconda   File: bioconductor_skeleton.py    License: MIT License 6 votes vote down vote up
def tarball_url(self):
        if not self._tarball_url:
            urls = [self.bioconductor_tarball_url,
                    self.bioarchive_url,
                    self.cargoport_url]
            for url in urls:
                if url is not None:
                    response = requests.head(url)
                    if response.status_code == 200:
                        self._tarball_url = url
                        return url

            logger.error(
                'No working URL for %s==%s in Bioconductor %s. '
                'Tried the following:\n\t' + '\n\t'.join(urls),
                self.package, self.version, self.bioc_version
            )

            if self._auto:
                find_best_bioc_version(self.package, self.version)

            if self._tarball_url is None:
                raise ValueError(
                    "No working URLs found for this version in any bioconductor version")
        return self._tarball_url 
Example 9
Project: Python-Scripts   Author: bing0o   File: subchecker.py    License: GNU General Public License v3.0 6 votes vote down vote up
def check(out, url):
	printer("Testing: " + url)
	url = 'http://' + url
	try:
		req = requests.head(url, timeout=10)
		scode = str(req.status_code)
		if scode.startswith("2"):
			print(green + "[+] "+scode+" | Found: " + end + "[ " + url + " ]")
		elif scode.startswith("3"):
			link = req.headers['Location']
			print(yellow + "[*] "+scode+" | Redirection: " + end + "[ " + url + " ]" + yellow + " -> " + end + "[ " + link + " ]")
		elif st.startswith("4"):
			print(blue+"[!] "+scode+" | Check: " + end + "[ " + url + " ]")

		if out != 'None':
			with open(out, 'a') as f:
				f.write(url+"\n")
				f.close()

		return True

	except Exception:
		return False 
Example 10
Project: QMusic   Author: dragondjf   File: downloadsongworker.py    License: GNU Lesser General Public License v2.1 6 votes vote down vote up
def support_continue(self, url):
        '''
            check support continue download or not
        '''
        headers = {'Range': 'bytes=0-4'}
        try:
            r = requests.head(url, headers=headers)
            if 'content-range' in r.headers:
                crange = r.headers['content-range']
                self.total = int(re.match(ur'^bytes 0-4/(\d+)$', crange).group(1))
                self.songObj.sizeUpdated.emit(self.total)
                return True
            else:
                self.total = 0
                return False
        except:
            logger.error(traceback.print_exc())
        try:
            self.total = int(r.headers['content-length'])
            self.songObj.sizeUpdated.emit(self.total)
        except:
            logger.error(traceback.print_exc())
            self.total = 0
        return False 
Example 11
Project: training_results_v0.6   Author: mlperf   File: test_forward.py    License: Apache License 2.0 6 votes vote down vote up
def _download(url, path, overwrite=False, sizecompare=False):
    ''' Download from internet'''
    if os.path.isfile(path) and not overwrite:
        if sizecompare:
            file_size = os.path.getsize(path)
            res_head = requests.head(url)
            res_get = requests.get(url, stream=True)
            if 'Content-Length' not in res_head.headers:
                res_get = urllib2.urlopen(url)
            urlfile_size = int(res_get.headers['Content-Length'])
            if urlfile_size != file_size:
                print("exist file got corrupted, downloading", path, " file freshly")
                _download(url, path, True, False)
                return
        print('File {} exists, skip.'.format(path))
        return
    print('Downloading from url {} to {}'.format(url, path))
    try:
        urllib.request.urlretrieve(url, path)
        print('')
    except:
        urllib.urlretrieve(url, path) 
Example 12
Project: bbotte.github.io   Author: bbotte   File: harborclient.py    License: Apache License 2.0 6 votes vote down vote up
def check_project_exist(self, project_name):
        result = False
        path = '%s://%s:%s/api/projects?project_name=%s' % (
            self.protocol, self.host, self.port,project_name)
        response = requests.head(path,
                                 cookies={'beegosessionID': self.session_id}, verify=False)
        if response.status_code == 200:
            result = True
            logger.debug(
                "Successfully check project exist, result: {}".format(result))
        elif response.status_code == 404:
            result = False
            logger.debug(
                "Successfully check project exist, result: {}".format(result))
        else:
            logger.error("Fail to check project exist")
        return result

    # POST /projects 
Example 13
Project: udata   Author: opendatateam   File: dcat.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def initialize(self):
        '''List all datasets for a given ...'''
        fmt = guess_format(self.source.url)
        # if format can't be guessed from the url
        # we fallback on the declared Content-Type
        if not fmt:
            response = requests.head(self.source.url)
            mime_type = response.headers.get('Content-Type', '').split(';', 1)[0]
            if not mime_type:
                msg = 'Unable to detect format from extension or mime type'
                raise ValueError(msg)
            fmt = guess_format(mime_type)
            if not fmt:
                msg = 'Unsupported mime type "{0}"'.format(mime_type)
                raise ValueError(msg)
        graph = self.parse_graph(self.source.url, fmt)
        self.job.data = {'graph': graph.serialize(format='json-ld', indent=None)} 
Example 14
Project: S3Scanner   Author: sa7mon   File: s3utils.py    License: MIT License 6 votes vote down vote up
def checkBucketWithoutCreds(bucketName, triesLeft=2):
    """ Does a simple GET request with the Requests library and interprets the results.
    bucketName - A domain name without protocol (http[s]) """

    if triesLeft == 0:
        return False

    bucketUrl = 'http://' + bucketName + '.s3.amazonaws.com'

    r = requests.head(bucketUrl)

    if r.status_code == 200:    # Successfully found a bucket!
        return True
    elif r.status_code == 403:  # Bucket exists, but we're not allowed to LIST it.
        return True
    elif r.status_code == 404:  # This is definitely not a valid bucket name.
        return False
    elif r.status_code == 503:
        return checkBucketWithoutCreds(bucketName, triesLeft - 1)
    else:
        raise ValueError("Got an unhandled status code back: " + str(r.status_code) + " for bucket: " + bucketName +
                         ". Please open an issue at: https://github.com/sa7mon/s3scanner/issues and include this info.") 
Example 15
Project: swarm   Author: a7vinx   File: dirsc.py    License: GNU General Public License v3.0 6 votes vote down vote up
def _scan_target_normal(self,target):
        try:
            r=requests.head(target)
            #it maybe a directory
            if self._check_eponymous_dir(r,target):
                return target+'/;'

            if self._check_exist_code(r.status_code):
                # check content
                r=requests.get(target)
                if self._check_exist_code(r.status_code):
                    for cur in self._not_exist_flag:
                        if r.text.find(cur)!=-1:
                            return ''
                    return target+';'
            return ''
        except Exception as e:
            return '' 
Example 16
Project: s3recon   Author: clarketm   File: s3recon.py    License: MIT License 6 votes vote down vote up
def bucket_exists(url, timeout):
    exists = False
    public = False

    try:
        res = requests.head(
            url,
            headers={"User-Agent": choice(useragent_list)},
            verify=False,
            timeout=timeout,
        )
        # TODO: handle redirects
        status_code = res.status_code
        exists = status_code != 404
        public = status_code == 200
    except RequestException:
        pass

    return exists, public 
Example 17
Project: datasploit   Author: dvopsway   File: username_gitlabdetails.py    License: GNU General Public License v3.0 6 votes vote down vote up
def main(username):
    # Use the username variable to do some stuff and return the data
    gitlabdetails = []
    url = "https://gitlab.com/" + username
    if (requests.head(url, verify=False).status_code == 200):
        response = requests.get(url)
        soup = BeautifulSoup(response.content, "lxml")
        handle= soup.find("span", {"class" : "middle-dot-divider"})
        if handle:
            name= soup.find("div", {"class" : "cover-title"})
            gitlabdetails.append("Name: " +name.text.strip())
            handle= soup.find("span", {"class" : "middle-dot-divider"})
            mydivs = soup.findAll("div", { "class" : "profile-link-holder middle-dot-divider" })
            for div in mydivs:
                q=div.find('a', href=True)
                if q:
                  gitlabdetails.append(q['href'].strip())
                elif div.find("i", {"class" : "fa fa-map-marker"}):
                  gitlabdetails.append("Location:" + div.text.strip())
                elif div.find("i", {"class" : "fa fa-briefcase"}):
                  gitlabdetails.append("Organisation: " + div.text.strip())

    return gitlabdetails 
Example 18
Project: bioservices   Author: cokelaer   File: biomart.py    License: GNU General Public License v3.0 5 votes vote down vote up
def _set_host(self, host):
        import requests
        secure = ''
        if self._secure:
            secure = 's'
        url = "http{}://{}/biomart/martservice".format(secure, host)
        request = requests.head(url)
        if request.status_code in [200]:
            self._host = host
            self.url = url
            self._init()
        else:
            self.logging.warning("host {} is not reachable ".format(host)) 
Example 19
Project: schema.data.gouv.fr   Author: etalab   File: main.py    License: MIT License 5 votes vote down vote up
def remote_available(self):
        if not self.git_url.startswith("http"):
            raise NotImplementedError(
                f"Can only check remote are available over HTTP. git_url is {git_url}"
            )

        try:
            r = requests.head(self.git_url, allow_redirects=True, timeout=15)
            return r.status_code == requests.codes.ok
        except requests.exceptions.RequestException:
            return False 
Example 20
Project: gradio-UI   Author: gradio-app   File: networking.py    License: Apache License 2.0 5 votes vote down vote up
def url_ok(url):
    try:
        r = requests.head(url)
        return r.status_code == 200
    except ConnectionError:
        return False 
Example 21
Project: gradio-UI   Author: gradio-app   File: networking.py    License: Apache License 2.0 5 votes vote down vote up
def url_ok(url):
    try:
        r = requests.head(url)
        return r.status_code == 200
    except ConnectionError:
        return False 
Example 22
Project: seqr   Author: macarthur-lab   File: download_utils.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def _get_remote_file_size(url):
    if url.startswith("http"):
        response = requests.head(url)
        return int(response.headers.get('Content-Length', '0'))
    else:
        return 0  # file size not yet implemented for FTP and other protocols 
Example 23
Project: Google-Images-Search   Author: arrrlo   File: google_api.py    License: MIT License 5 votes vote down vote up
def search(self, params, cache_discovery=False):
        """Search for images and returns
        them using generator object
        :param params: search params
        :param cache_discovery whether or not to cache the discovery doc
        :return: image list
        """

        search_params = self._search_params(params)
        res = self._query_google_api(search_params, cache_discovery)

        for image in res.get('items', []):
            if self._fetch_resize_save.validate_images:
                try:
                    response = requests.head(image['link'], timeout=5)
                    content_length = response.headers.get('Content-Length')
                    content_type = response.headers.get('Content-Type', '')

                    # check if the url is valid
                    if response.status_code == 200 and \
                            'image' in content_type and content_length:

                        # calculate download chunk size based on image size
                        self._fetch_resize_save.set_chunk_size(
                            image['link'], content_length
                        )

                        # if everything is ok, yield it out
                        yield image['link']

                except requests.exceptions.ConnectTimeout:
                    pass
                except requests.exceptions.ReadTimeout:
                    pass
                except requests.exceptions.SSLError:
                    pass 
Example 24
Project: dephell   Author: dephell   File: package_bug.py    License: MIT License 5 votes vote down vote up
def _get_url(links: Dict[str, str]) -> Optional[str]:
        # try to find githab or gitlub url and use it as a bug tracker
        for url in links.values():
            if not url.startswith('http'):
                url = 'https://' + url
            parsed = urlparse(url)
            if parsed.hostname not in ('github.com', 'gitlab.com', 'bitbucket.org'):
                continue

            # build URL
            parts = parsed.path.strip('/').split('/')
            if len(parts) < 2:
                continue
            url = 'https://{}/{}/{}/issues/new'.format(parsed.hostname, *parts)

            # check that issues aren't disabled for the project
            response = requests.head(url)
            if response.status_code == 404:
                continue

            return url

        # try to find custom bug tracker by name
        for name, url in links.items():
            if 'tracker' not in name.lower():
                continue
            if not url.startswith('http'):
                url = 'https://' + url
            return url

        return None 
Example 25
Project: dephell   Author: dephell   File: _repos.py    License: MIT License 5 votes vote down vote up
def _has_api(url: str) -> bool:
    if urlparse(url).hostname in ('pypi.org', 'python.org', 'test.pypi.org'):
        return True
    full_url = urljoin(url, 'dephell/json/')
    try:
        response = requests.head(full_url)
    except (SSLError, ConnectionError):
        return False
    return response.status_code < 400 
Example 26
Project: ThreatIngestor   Author: InQuest   File: web.py    License: GNU General Public License v2.0 5 votes vote down vote up
def run(self, saved_state):
        # Read saved state and set HTTP headers.
        headers = {}
        if saved_state:
            # If both last modified and etag, set both.
            # Otherwise just interpret the whole field as last modified.
            last_modified = saved_state

            if len(saved_state.split(';')) == 2:
                last_modified, etag = saved_state.split(';')
                headers['If-None-Match'] = etag

            headers['If-Modified-Since'] = last_modified

        # Send head first to check 304.
        response = requests.head(self.url, headers=headers)

        # If not modified, return immediately.
        if response.status_code == 304:
            return saved_state, []

        # Otherwise, do the full request.
        response = requests.get(self.url, headers=headers)

        # Form saved state.
        last_modified = response.headers.get('Last-Modified')
        etag = response.headers.get('Etag')

        if etag:
            saved_state = ';'.join([str(last_modified), etag])
        else:
            saved_state = last_modified

        # Process text.
        artifact_list = self.process_element(response.text, self.url, include_nonobfuscated=True)

        return saved_state, artifact_list 
Example 27
Project: astroNN   Author: henrysky   File: test_datasets.py    License: MIT License 5 votes vote down vote up
def test_galaxy10(self):
        # make sure galaxy10 exists on Bovy's server

        r = requests.head(_G10_ORIGIN, allow_redirects=True)
        self.assertEqual(r.status_code, 200)
        r.close()

        galaxy10cls_lookup(0)
        self.assertRaises(ValueError, galaxy10cls_lookup, 11)
        galaxy10_confusion(np.ones((10,10))) 
Example 28
Project: insightconnect-plugins   Author: rapid7   File: helpers.py    License: MIT License 5 votes vote down vote up
def head_url(url):
    '''Returns a request HEAD object given a URL as a string'''
    return requests.head(add_schema(url)) 
Example 29
Project: insightconnect-plugins   Author: rapid7   File: connection.py    License: MIT License 5 votes vote down vote up
def connect(self, params={}):
        self.logger.info("Connect: Connecting..")
        url = params.get('url')
        try:
            r = requests.head(url)
            if r.status_code != 200:
                self.logger.error('Logstash: Connect: error %s', params)
        except requests.ConnectionError:
            self.logger.error('Logstash: Connect: error %s', params)
            raise Exception('Logstash: Connect: connection failed')

        self.url = url 
Example 30
Project: celerystalk   Author: sethsec   File: generic_urlextract.py    License: MIT License 5 votes vote down vote up
def check_if_page_exists(url):
    try:
        response = requests.head(url, timeout=5, verify=False)
        status_code = response.status_code
    except requests.exceptions.ConnectionError:
        status_code = 999
    return status_code