Python requests.head() Examples

The following are 30 code examples of requests.head(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module requests , or try the search function .
Example #1
Source File: username_gitlabdetails.py    From datasploit with GNU General Public License v3.0 6 votes vote down vote up
def main(username):
    # Use the username variable to do some stuff and return the data
    gitlabdetails = []
    url = "https://gitlab.com/" + username
    if (requests.head(url, verify=False).status_code == 200):
        response = requests.get(url)
        soup = BeautifulSoup(response.content, "lxml")
        handle= soup.find("span", {"class" : "middle-dot-divider"})
        if handle:
            name= soup.find("div", {"class" : "cover-title"})
            gitlabdetails.append("Name: " +name.text.strip())
            handle= soup.find("span", {"class" : "middle-dot-divider"})
            mydivs = soup.findAll("div", { "class" : "profile-link-holder middle-dot-divider" })
            for div in mydivs:
                q=div.find('a', href=True)
                if q:
                  gitlabdetails.append(q['href'].strip())
                elif div.find("i", {"class" : "fa fa-map-marker"}):
                  gitlabdetails.append("Location:" + div.text.strip())
                elif div.find("i", {"class" : "fa fa-briefcase"}):
                  gitlabdetails.append("Organisation: " + div.text.strip())

    return gitlabdetails 
Example #2
Source File: selescrape.py    From anime-downloader with The Unlicense 6 votes vote down vote up
def status_select(driver, url, status='hide'):
    '''
    For now it doesnt do what its name suggests, 
    I have planned to add a status reporter of the http response code.
    This part of the code is not removed because it is part of its core.
    Treat it like it isnt here.
    '''
    try:
        if status == 'hide':
            driver.get(url)
        elif status == 'show':
            r = requests.head(url)
            if r.status_code == 503:
                raise RuntimeError("This website's sevice is unavailable or has cloudflare on.")
            driver.get(url)
            return r.status_code
        else:
            driver.get(url)
    except requests.ConnectionError:
        raise RuntimeError("Failed to establish a connection using the requests library.") 
Example #3
Source File: display_video_360.py    From orchestra with Apache License 2.0 6 votes vote down vote up
def _download_report(source_url, destination_file, chunk_size):
        response = requests.head(source_url)
        content_length = int(response.headers['Content-Length'])

        start_byte = 0
        while start_byte < content_length:
            end_byte = start_byte + chunk_size - 1
            if end_byte >= content_length:
                end_byte = content_length - 1

            headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)}
            response = requests.get(source_url, stream=True, headers=headers)
            chunk = response.raw.read()
            destination_file.write(chunk)
            start_byte = end_byte + 1
        destination_file.close() 
Example #4
Source File: subchecker.py    From Python-Scripts with GNU General Public License v3.0 6 votes vote down vote up
def check(out, url):
	printer("Testing: " + url)
	url = 'http://' + url
	try:
		req = requests.head(url, timeout=10)
		scode = str(req.status_code)
		if scode.startswith("2"):
			print(green + "[+] "+scode+" | Found: " + end + "[ " + url + " ]")
		elif scode.startswith("3"):
			link = req.headers['Location']
			print(yellow + "[*] "+scode+" | Redirection: " + end + "[ " + url + " ]" + yellow + " -> " + end + "[ " + link + " ]")
		elif st.startswith("4"):
			print(blue+"[!] "+scode+" | Check: " + end + "[ " + url + " ]")

		if out != 'None':
			with open(out, 'a') as f:
				f.write(url+"\n")
				f.close()

		return True

	except Exception:
		return False 
Example #5
Source File: downloadsongworker.py    From QMusic with GNU Lesser General Public License v2.1 6 votes vote down vote up
def support_continue(self, url):
        '''
            check support continue download or not
        '''
        headers = {'Range': 'bytes=0-4'}
        try:
            r = requests.head(url, headers=headers)
            if 'content-range' in r.headers:
                crange = r.headers['content-range']
                self.total = int(re.match(ur'^bytes 0-4/(\d+)$', crange).group(1))
                self.songObj.sizeUpdated.emit(self.total)
                return True
            else:
                self.total = 0
                return False
        except:
            logger.error(traceback.print_exc())
        try:
            self.total = int(r.headers['content-length'])
            self.songObj.sizeUpdated.emit(self.total)
        except:
            logger.error(traceback.print_exc())
            self.total = 0
        return False 
Example #6
Source File: crawlbox.py    From CrawlBox with The Unlicense 6 votes vote down vote up
def checkUrl(url):

    # check
    try:
        ress1 = requests.head(url , allow_redirects=True)

        if url != ress1.url:
            return "Maybe you should use ;"+ress1.url
        else:
            ress = requests.get(url)
            code = ress.status_code
            if (code == 200):
                return True
            else:
               return False
    except requests.exceptions.ConnectionError:
        return "Try a different url please"
    except requests.exceptions.MissingSchema:
        return "Try a different url please"
    except:
        return False

## make input work both in python3 and 2 
Example #7
Source File: generate.py    From Awesome-Mobile-UI with Apache License 2.0 6 votes vote down vote up
def download_file(url, filename):
    # NOTE the stream=True parameter
    # response = requests.head(url)

    # if "Content-Length" in response.headers:
    #     print("length---" + response.headers["Content-Length"])
    # if "Content-Length" in response.headers and int(response.headers["Content-Length"]) > 100000:
    #     r = requests.get(url, stream=True)
    #     with open(filename, 'wb') as f:
    #         for chunk in r.iter_content(chunk_size=1024):
    #             if chunk:  # filter out keep-alive new chunks
    #                 f.write(chunk)
    #                 f.flush()
    # return filename
    r = requests.get(url, stream=True)
    if len(r.content) > 50000:
        # print("img url----" + url + "     " + str(len(r.content)))
        with open(filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
        return True
    else:
        return False 
Example #8
Source File: bioconductor_skeleton.py    From bioconda-utils with MIT License 6 votes vote down vote up
def tarball_url(self):
        if not self._tarball_url:
            urls = [self.bioconductor_tarball_url,
                    self.bioarchive_url,
                    self.cargoport_url]
            for url in urls:
                if url is not None:
                    response = requests.head(url)
                    if response.status_code == 200:
                        self._tarball_url = url
                        return url

            logger.error(
                'No working URL for %s==%s in Bioconductor %s. '
                'Tried the following:\n\t' + '\n\t'.join(urls),
                self.package, self.version, self.bioc_version
            )

            if self._auto:
                find_best_bioc_version(self.package, self.version)

            if self._tarball_url is None:
                raise ValueError(
                    "No working URLs found for this version in any bioconductor version")
        return self._tarball_url 
Example #9
Source File: test_forward.py    From training_results_v0.6 with Apache License 2.0 6 votes vote down vote up
def _download(url, path, overwrite=False, sizecompare=False):
    ''' Download from internet'''
    if os.path.isfile(path) and not overwrite:
        if sizecompare:
            file_size = os.path.getsize(path)
            res_head = requests.head(url)
            res_get = requests.get(url, stream=True)
            if 'Content-Length' not in res_head.headers:
                res_get = urllib2.urlopen(url)
            urlfile_size = int(res_get.headers['Content-Length'])
            if urlfile_size != file_size:
                print("exist file got corrupted, downloading", path, " file freshly")
                _download(url, path, True, False)
                return
        print('File {} exists, skip.'.format(path))
        return
    print('Downloading from url {} to {}'.format(url, path))
    try:
        urllib.request.urlretrieve(url, path)
        print('')
    except:
        urllib.urlretrieve(url, path) 
Example #10
Source File: harborclient.py    From bbotte.github.io with Apache License 2.0 6 votes vote down vote up
def check_project_exist(self, project_name):
        result = False
        path = '%s://%s:%s/api/projects?project_name=%s' % (
            self.protocol, self.host, self.port,project_name)
        response = requests.head(path,
                                 cookies={'beegosessionID': self.session_id}, verify=False)
        if response.status_code == 200:
            result = True
            logger.debug(
                "Successfully check project exist, result: {}".format(result))
        elif response.status_code == 404:
            result = False
            logger.debug(
                "Successfully check project exist, result: {}".format(result))
        else:
            logger.error("Fail to check project exist")
        return result

    # POST /projects 
Example #11
Source File: hello.py    From PythonEXE with MIT License 6 votes vote down vote up
def main():
    print('hello world')
    print('contacting google.com...')
    r = requests.head("https://www.google.com")
    print("status code:", r.status_code)
    print("PATH:", os.getenv("PATH"))
    result = helper.add(1, 1)
    print(f"1 + 1 = {result}")
    try:
        with open(Path(BASE_DIR, "input.txt")) as f:
            content = f.read().strip()
        print(content)
    except IOError:
        print("Warning: input.txt was not found")
    #
    fname = Path(BASE_DIR, "output.txt")
    print("writing to the following file:", fname)
    with open(fname, "w") as g:
        print("writing to a file", file=g)

############################################################################## 
Example #12
Source File: dcat.py    From udata with GNU Affero General Public License v3.0 6 votes vote down vote up
def initialize(self):
        '''List all datasets for a given ...'''
        fmt = guess_format(self.source.url)
        # if format can't be guessed from the url
        # we fallback on the declared Content-Type
        if not fmt:
            response = requests.head(self.source.url)
            mime_type = response.headers.get('Content-Type', '').split(';', 1)[0]
            if not mime_type:
                msg = 'Unable to detect format from extension or mime type'
                raise ValueError(msg)
            fmt = guess_format(mime_type)
            if not fmt:
                msg = 'Unsupported mime type "{0}"'.format(mime_type)
                raise ValueError(msg)
        graph = self.parse_graph(self.source.url, fmt)
        self.job.data = {'graph': graph.serialize(format='json-ld', indent=None)} 
Example #13
Source File: bioconductor_skeleton.py    From bioconda-utils with MIT License 6 votes vote down vote up
def cargoport_url(self):
        """
        Returns the Galaxy cargo-port URL for this version of this package, if
        it exists.
        """
        url = cargoport_url(self.package, self.version, self.bioc_version)
        response = requests.head(url)
        if response.status_code == 404:
            # This is expected if this is a new package or an updated version.
            # Cargo Port will archive a working URL upon merging
            return
        elif response.status_code == 200:
            return url
        else:
            raise PageNotFoundError(
                "Unexpected error: {0.status_code} ({0.reason})".format(response)) 
Example #14
Source File: s3utils.py    From S3Scanner with MIT License 6 votes vote down vote up
def checkBucketWithoutCreds(bucketName, triesLeft=2):
    """ Does a simple GET request with the Requests library and interprets the results.
    bucketName - A domain name without protocol (http[s]) """

    if triesLeft == 0:
        return False

    bucketUrl = 'http://' + bucketName + '.s3.amazonaws.com'

    r = requests.head(bucketUrl)

    if r.status_code == 200:    # Successfully found a bucket!
        return True
    elif r.status_code == 403:  # Bucket exists, but we're not allowed to LIST it.
        return True
    elif r.status_code == 404:  # This is definitely not a valid bucket name.
        return False
    elif r.status_code == 503:
        return checkBucketWithoutCreds(bucketName, triesLeft - 1)
    else:
        raise ValueError("Got an unhandled status code back: " + str(r.status_code) + " for bucket: " + bucketName +
                         ". Please open an issue at: https://github.com/sa7mon/s3scanner/issues and include this info.") 
Example #15
Source File: dirsc.py    From swarm with GNU General Public License v3.0 6 votes vote down vote up
def _scan_target_normal(self,target):
        try:
            r=requests.head(target)
            #it maybe a directory
            if self._check_eponymous_dir(r,target):
                return target+'/;'

            if self._check_exist_code(r.status_code):
                # check content
                r=requests.get(target)
                if self._check_exist_code(r.status_code):
                    for cur in self._not_exist_flag:
                        if r.text.find(cur)!=-1:
                            return ''
                    return target+';'
            return ''
        except Exception as e:
            return '' 
Example #16
Source File: api.py    From python-wp with MIT License 6 votes vote down vote up
def _get_wp_api_url(self, url):
        """
        Private function for finding the WP-API URL.

        Arguments
        ---------

        url : str
            WordPress instance URL.
        """
        resp = requests.head(url)

        # Search the Links for rel="https://api.w.org/".
        wp_api_rel = resp.links.get('https://api.w.org/')

        if wp_api_rel:
            return wp_api_rel['url']
        else:
            # TODO: Rasie a better exception to the rel doesn't exist.
            raise Exception 
Example #17
Source File: s3recon.py    From s3recon with MIT License 6 votes vote down vote up
def bucket_exists(url, timeout):
    exists = False
    public = False

    try:
        res = requests.head(
            url,
            headers={"User-Agent": choice(useragent_list)},
            verify=False,
            timeout=timeout,
        )
        # TODO: handle redirects
        status_code = res.status_code
        exists = status_code != 404
        public = status_code == 200
    except RequestException:
        pass

    return exists, public 
Example #18
Source File: util.py    From sumgram with MIT License 5 votes vote down vote up
def nlpIsServerOn(addr='http://localhost:9000'):

    try:
        response = requests.head(addr)
        
        if( response.status_code == 200 ):
            return True
        else:
            return False

    except:
        genericErrorInfo()

    return False 
Example #19
Source File: tus.py    From tus.py with MIT License 5 votes vote down vote up
def _get_offset(file_endpoint, headers=None):
    logger.info("Getting offset")

    h = {"Tus-Resumable": TUS_VERSION}

    if headers:
        h.update(headers)

    response = requests.head(file_endpoint, headers=h)
    response.raise_for_status()

    offset = int(response.headers["Upload-Offset"])
    logger.info("offset=%i", offset)
    return offset 
Example #20
Source File: utilities.py    From overcast-sonos with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def final_redirect_url(url):
    redirected_url = requests.head(url, allow_redirects=True).url
    if url != redirected_url:
        log.debug('''Redirected url '%s' to '%s'.''', url, redirected_url)
    return redirected_url 
Example #21
Source File: bioconductor_skeleton.py    From bioconda-utils with MIT License 5 votes vote down vote up
def find_best_bioc_version(package, version):
    """
    Given a package version number, identifies which BioC version[s] it is in
    and returns the latest.

    If the package is not found, raises a PackageNotFound error.

    Parameters
    ----------
    package :
        Case-sensitive Bioconductor package name

    version :
        Bioconductor package version

    Returns None if no valid package found.
    """
    for bioc_version in bioconductor_versions():
        for kind, func in zip(
            ('package', 'data'),
            (
                bioconductor_tarball_url, bioconductor_annotation_data_url,
                bioconductor_experiment_data_url,
            ),
        ):
            url = func(package, version, bioc_version)
            if requests.head(url).status_code == 200:
                logger.debug('success: %s', url)
                logger.info(
                    'A working URL for %s==%s was identified for Bioconductor version %s: %s',
                    package, version, bioc_version, url)
                found_version = bioc_version
                return found_version
            else:
                logger.debug('missing: %s', url)
    raise PackageNotFoundError(
        "Cannot find any Bioconductor versions for {0}=={1}".format(package, version)
    ) 
Example #22
Source File: check_broken_links.py    From psutil with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def validate_url(url):
    """Validate the URL by attempting an HTTP connection.
    Makes an HTTP-HEAD request for each URL.
    """
    try:
        res = requests.head(url, timeout=REQUEST_TIMEOUT)
        # some websites deny 503, like Microsoft
        # and some send 401, like Apple, observations
        if (not res.ok) and (res.status_code in RETRY_STATUSES):
            res = requests.get(url, timeout=REQUEST_TIMEOUT)
        return res.ok
    except requests.exceptions.RequestException:
        return False 
Example #23
Source File: StatusCode.py    From Python-Scripts with GNU General Public License v3.0 5 votes vote down vote up
def check(out, url):
	#print("hell")
	printer("Testing: " + url)
	link = 'http://' + url
	try:
		req = requests.head(link, timeout=10)
		length = str(req.headers['Content-Length'])
		scode = str(req.status_code)
		if scode.startswith("2"):
			print(blue+"["+bold+green+str(scode)+end+blue+"]"+end+" | "+str(url)+length)
		elif scode.startswith("3"):
			if req.headers['Location'].startswith("https://"+url):
				print(blue+"["+bold+yellow+str(scode)+end+blue+"]"+end+" | "+str(url)+" - HTTPS | "+length)
			else:
				print(blue+"["+bold+yellow+str(scode)+end+blue+"]"+end+" | "+str(url)+" | "+req.headers['Location']+" | "+length)
		elif scode.startswith("4"):
			print(blue+"["+bold+red+str(scode)+end+blue+"]"+end+" | "+str(url)+" | "+length)
		else:
			print(blue+"["+end+str(scode)+blue+"]"+end+" | "+str(url)+" | "+length)

		if out != 'None':
			with open(out, 'a') as f:
				f.write(str(scode)+" | "+url+" | "+length+"\n")
				f.close()

		return True

	except Exception:
		return False 
Example #24
Source File: dirbrute.py    From Python-Scripts with GNU General Public License v3.0 5 votes vote down vote up
def checkstatus(domain, url):
	if url == "" or url == " ":
		pass
	elif url.startswith("#"):
		pass
	elif len(url) > 30:
		pass

	else:
		printer("Testing: " + domain + url)
		#time.sleep(1)
		try:
			link = domain + url
			req = requests.head(link)
			st = str(req.status_code)
			if st.startswith("2"):
				print(green + "[+] "+st+" | Found: " + end + "[ " + url + " ]" + "                                                   \r")
			elif st.startswith("3"):
				link = req.headers['Location']
				#link = req.url
				print(yellow + "[*] "+st+" | Redirection From: " + end + "[ " + url + " ]" + yellow + " -> " + end + "[ " + link + " ]" + "                                         \r")

			elif st.startswith("1"):
				print(green + "[+] "+st+" | Found: " + end + "[ " + url + " ]" + "                                                   \r")
			elif st.startswith("4"):
				if st != '404':
					print(blue+"[!] "+st+" | Found: " + end + "[ " + url + " ]" + "                                                   \r")

					#writer(link,'up')
			
			return True
			
		except Exception:
			#writer(url,'down')
			return False 
Example #25
Source File: tasks.py    From sregistry-cli with Mozilla Public License 2.0 5 votes vote down vote up
def download(url, file_name, headers=None, show_progress=True):
    """stream to a temporary file, rename on successful completion

        Parameters
        ==========
        file_name: the file name to stream to
        url: the url to stream from
        headers: additional headers to add
    """

    fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name))
    os.close(fd)

    if DISABLE_SSL_CHECK is True:
        bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::")

    verify = not DISABLE_SSL_CHECK

    # Does the url being requested exist?
    if requests.head(url, verify=verify).status_code in [200, 401]:
        response = stream(url, headers=headers, stream_to=tmp_file)

        if isinstance(response, HTTPError):
            bot.exit("Error downloading %s, exiting." % url)

        shutil.move(tmp_file, file_name)
    else:
        bot.error("Invalid url or permissions %s" % url)
    return file_name 
Example #26
Source File: http.py    From sregistry-cli with Mozilla Public License 2.0 5 votes vote down vote up
def download(self, url, file_name, headers=None, show_progress=True):

    """stream to a temporary file, rename on successful completion

        Parameters
        ==========
        file_name: the file name to stream to
        url: the url to stream from
        headers: additional headers to add
        force: If the final image exists, don't overwrite
        show_progress: boolean to show progress bar
    """

    tmp_file = get_tmpfile(prefix="%s.tmp." % file_name)

    # Should we verify the request?
    verify = self._verify()

    # Check here if exists
    if requests.head(url, verify=verify).status_code in [200, 401]:
        response = self.stream(
            url, headers=headers, stream_to=tmp_file, show_progress=show_progress
        )

        if isinstance(response, HTTPError):
            bot.exit("Error downloading %s, exiting." % url)

        shutil.move(tmp_file, file_name)
    else:
        bot.error("Invalid url or permissions %s" % url)
    return file_name 
Example #27
Source File: bannerscan.py    From fuzzdb-collect with GNU General Public License v3.0 5 votes vote down vote up
def run(self):
        result[self.ip] = dict()
        for port in PORTS:
            url_pre = "https://" if port == 443 else "http://"
            site = url_pre + self.ip + ":" + str(port)
            try:
                print ("[*] %s\r" % (site[0:60].ljust(60, " "))),
                resp = requests.head(site,
                                     allow_redirects = False,
                                     timeout=self.timeout,
                                     headers=self.headers
                )
                result[self.ip][port] = dict()

            except Exception, e:
                pass

            else:
                result[self.ip][port]["headers"] = resp.headers
                result[self.ip][port]["available"] = list()

                for path in PATHS:
                    try:
                        url = site + path
                        print ("[*] %s\r" % (url[0:60].ljust(60, " "))),
                        resp = self.req.get(url,
                                            allow_redirects = False,
                                            timeout=self.timeout,
                                            headers=self.headers
                        )

                    except Exception, e:
                        pass
                    else:
                        if resp.status_code in [200, 406, 401, 403, 500]:
                            r = re.findall("<title>([\s\S]+?)</title>", resp.content)
                            title = lambda r : r and r[0] or ""
                            result[self.ip][port]["available"].append((title(r), url, resp.status_code)) 
Example #28
Source File: http.py    From sregistry-cli with Mozilla Public License 2.0 5 votes vote down vote up
def head(self, url):
    """head request, typically used for status code retrieval, etc.
    """
    bot.debug("HEAD %s" % url)
    return self._call(url, func=requests.head) 
Example #29
Source File: bioconductor_skeleton.py    From bioconda-utils with MIT License 5 votes vote down vote up
def bioarchive_url(self):
        """
        Returns the bioaRchive URL if one exists for this version of this
        package, otherwise returns None.
        """
        url = bioarchive_url(self.package, self.version, self.bioc_version)
        response = requests.head(url)
        if response.status_code == 200:
            return url 
Example #30
Source File: bioconductor_skeleton.py    From bioconda-utils with MIT License 5 votes vote down vote up
def bioconductor_tarball_url(self):
        """
        Return the url to the tarball from the bioconductor site.
        """
        url = os.path.join(base_url, self.bioc_version, self.packages[self.package]['URLprefix'], self.packages[self.package]['source.ver'])
        response = requests.head(url)
        if response.status_code == 200:
            return url