Python requests.head() Examples
The following are 30
code examples of requests.head().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
requests
, or try the search function
.
Example #1
Source File: username_gitlabdetails.py From datasploit with GNU General Public License v3.0 | 6 votes |
def main(username): # Use the username variable to do some stuff and return the data gitlabdetails = [] url = "https://gitlab.com/" + username if (requests.head(url, verify=False).status_code == 200): response = requests.get(url) soup = BeautifulSoup(response.content, "lxml") handle= soup.find("span", {"class" : "middle-dot-divider"}) if handle: name= soup.find("div", {"class" : "cover-title"}) gitlabdetails.append("Name: " +name.text.strip()) handle= soup.find("span", {"class" : "middle-dot-divider"}) mydivs = soup.findAll("div", { "class" : "profile-link-holder middle-dot-divider" }) for div in mydivs: q=div.find('a', href=True) if q: gitlabdetails.append(q['href'].strip()) elif div.find("i", {"class" : "fa fa-map-marker"}): gitlabdetails.append("Location:" + div.text.strip()) elif div.find("i", {"class" : "fa fa-briefcase"}): gitlabdetails.append("Organisation: " + div.text.strip()) return gitlabdetails
Example #2
Source File: selescrape.py From anime-downloader with The Unlicense | 6 votes |
def status_select(driver, url, status='hide'): ''' For now it doesnt do what its name suggests, I have planned to add a status reporter of the http response code. This part of the code is not removed because it is part of its core. Treat it like it isnt here. ''' try: if status == 'hide': driver.get(url) elif status == 'show': r = requests.head(url) if r.status_code == 503: raise RuntimeError("This website's sevice is unavailable or has cloudflare on.") driver.get(url) return r.status_code else: driver.get(url) except requests.ConnectionError: raise RuntimeError("Failed to establish a connection using the requests library.")
Example #3
Source File: display_video_360.py From orchestra with Apache License 2.0 | 6 votes |
def _download_report(source_url, destination_file, chunk_size): response = requests.head(source_url) content_length = int(response.headers['Content-Length']) start_byte = 0 while start_byte < content_length: end_byte = start_byte + chunk_size - 1 if end_byte >= content_length: end_byte = content_length - 1 headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)} response = requests.get(source_url, stream=True, headers=headers) chunk = response.raw.read() destination_file.write(chunk) start_byte = end_byte + 1 destination_file.close()
Example #4
Source File: subchecker.py From Python-Scripts with GNU General Public License v3.0 | 6 votes |
def check(out, url): printer("Testing: " + url) url = 'http://' + url try: req = requests.head(url, timeout=10) scode = str(req.status_code) if scode.startswith("2"): print(green + "[+] "+scode+" | Found: " + end + "[ " + url + " ]") elif scode.startswith("3"): link = req.headers['Location'] print(yellow + "[*] "+scode+" | Redirection: " + end + "[ " + url + " ]" + yellow + " -> " + end + "[ " + link + " ]") elif st.startswith("4"): print(blue+"[!] "+scode+" | Check: " + end + "[ " + url + " ]") if out != 'None': with open(out, 'a') as f: f.write(url+"\n") f.close() return True except Exception: return False
Example #5
Source File: downloadsongworker.py From QMusic with GNU Lesser General Public License v2.1 | 6 votes |
def support_continue(self, url): ''' check support continue download or not ''' headers = {'Range': 'bytes=0-4'} try: r = requests.head(url, headers=headers) if 'content-range' in r.headers: crange = r.headers['content-range'] self.total = int(re.match(ur'^bytes 0-4/(\d+)$', crange).group(1)) self.songObj.sizeUpdated.emit(self.total) return True else: self.total = 0 return False except: logger.error(traceback.print_exc()) try: self.total = int(r.headers['content-length']) self.songObj.sizeUpdated.emit(self.total) except: logger.error(traceback.print_exc()) self.total = 0 return False
Example #6
Source File: crawlbox.py From CrawlBox with The Unlicense | 6 votes |
def checkUrl(url): # check try: ress1 = requests.head(url , allow_redirects=True) if url != ress1.url: return "Maybe you should use ;"+ress1.url else: ress = requests.get(url) code = ress.status_code if (code == 200): return True else: return False except requests.exceptions.ConnectionError: return "Try a different url please" except requests.exceptions.MissingSchema: return "Try a different url please" except: return False ## make input work both in python3 and 2
Example #7
Source File: generate.py From Awesome-Mobile-UI with Apache License 2.0 | 6 votes |
def download_file(url, filename): # NOTE the stream=True parameter # response = requests.head(url) # if "Content-Length" in response.headers: # print("length---" + response.headers["Content-Length"]) # if "Content-Length" in response.headers and int(response.headers["Content-Length"]) > 100000: # r = requests.get(url, stream=True) # with open(filename, 'wb') as f: # for chunk in r.iter_content(chunk_size=1024): # if chunk: # filter out keep-alive new chunks # f.write(chunk) # f.flush() # return filename r = requests.get(url, stream=True) if len(r.content) > 50000: # print("img url----" + url + " " + str(len(r.content))) with open(filename, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() return True else: return False
Example #8
Source File: bioconductor_skeleton.py From bioconda-utils with MIT License | 6 votes |
def tarball_url(self): if not self._tarball_url: urls = [self.bioconductor_tarball_url, self.bioarchive_url, self.cargoport_url] for url in urls: if url is not None: response = requests.head(url) if response.status_code == 200: self._tarball_url = url return url logger.error( 'No working URL for %s==%s in Bioconductor %s. ' 'Tried the following:\n\t' + '\n\t'.join(urls), self.package, self.version, self.bioc_version ) if self._auto: find_best_bioc_version(self.package, self.version) if self._tarball_url is None: raise ValueError( "No working URLs found for this version in any bioconductor version") return self._tarball_url
Example #9
Source File: test_forward.py From training_results_v0.6 with Apache License 2.0 | 6 votes |
def _download(url, path, overwrite=False, sizecompare=False): ''' Download from internet''' if os.path.isfile(path) and not overwrite: if sizecompare: file_size = os.path.getsize(path) res_head = requests.head(url) res_get = requests.get(url, stream=True) if 'Content-Length' not in res_head.headers: res_get = urllib2.urlopen(url) urlfile_size = int(res_get.headers['Content-Length']) if urlfile_size != file_size: print("exist file got corrupted, downloading", path, " file freshly") _download(url, path, True, False) return print('File {} exists, skip.'.format(path)) return print('Downloading from url {} to {}'.format(url, path)) try: urllib.request.urlretrieve(url, path) print('') except: urllib.urlretrieve(url, path)
Example #10
Source File: harborclient.py From bbotte.github.io with Apache License 2.0 | 6 votes |
def check_project_exist(self, project_name): result = False path = '%s://%s:%s/api/projects?project_name=%s' % ( self.protocol, self.host, self.port,project_name) response = requests.head(path, cookies={'beegosessionID': self.session_id}, verify=False) if response.status_code == 200: result = True logger.debug( "Successfully check project exist, result: {}".format(result)) elif response.status_code == 404: result = False logger.debug( "Successfully check project exist, result: {}".format(result)) else: logger.error("Fail to check project exist") return result # POST /projects
Example #11
Source File: hello.py From PythonEXE with MIT License | 6 votes |
def main(): print('hello world') print('contacting google.com...') r = requests.head("https://www.google.com") print("status code:", r.status_code) print("PATH:", os.getenv("PATH")) result = helper.add(1, 1) print(f"1 + 1 = {result}") try: with open(Path(BASE_DIR, "input.txt")) as f: content = f.read().strip() print(content) except IOError: print("Warning: input.txt was not found") # fname = Path(BASE_DIR, "output.txt") print("writing to the following file:", fname) with open(fname, "w") as g: print("writing to a file", file=g) ##############################################################################
Example #12
Source File: dcat.py From udata with GNU Affero General Public License v3.0 | 6 votes |
def initialize(self): '''List all datasets for a given ...''' fmt = guess_format(self.source.url) # if format can't be guessed from the url # we fallback on the declared Content-Type if not fmt: response = requests.head(self.source.url) mime_type = response.headers.get('Content-Type', '').split(';', 1)[0] if not mime_type: msg = 'Unable to detect format from extension or mime type' raise ValueError(msg) fmt = guess_format(mime_type) if not fmt: msg = 'Unsupported mime type "{0}"'.format(mime_type) raise ValueError(msg) graph = self.parse_graph(self.source.url, fmt) self.job.data = {'graph': graph.serialize(format='json-ld', indent=None)}
Example #13
Source File: bioconductor_skeleton.py From bioconda-utils with MIT License | 6 votes |
def cargoport_url(self): """ Returns the Galaxy cargo-port URL for this version of this package, if it exists. """ url = cargoport_url(self.package, self.version, self.bioc_version) response = requests.head(url) if response.status_code == 404: # This is expected if this is a new package or an updated version. # Cargo Port will archive a working URL upon merging return elif response.status_code == 200: return url else: raise PageNotFoundError( "Unexpected error: {0.status_code} ({0.reason})".format(response))
Example #14
Source File: s3utils.py From S3Scanner with MIT License | 6 votes |
def checkBucketWithoutCreds(bucketName, triesLeft=2): """ Does a simple GET request with the Requests library and interprets the results. bucketName - A domain name without protocol (http[s]) """ if triesLeft == 0: return False bucketUrl = 'http://' + bucketName + '.s3.amazonaws.com' r = requests.head(bucketUrl) if r.status_code == 200: # Successfully found a bucket! return True elif r.status_code == 403: # Bucket exists, but we're not allowed to LIST it. return True elif r.status_code == 404: # This is definitely not a valid bucket name. return False elif r.status_code == 503: return checkBucketWithoutCreds(bucketName, triesLeft - 1) else: raise ValueError("Got an unhandled status code back: " + str(r.status_code) + " for bucket: " + bucketName + ". Please open an issue at: https://github.com/sa7mon/s3scanner/issues and include this info.")
Example #15
Source File: dirsc.py From swarm with GNU General Public License v3.0 | 6 votes |
def _scan_target_normal(self,target): try: r=requests.head(target) #it maybe a directory if self._check_eponymous_dir(r,target): return target+'/;' if self._check_exist_code(r.status_code): # check content r=requests.get(target) if self._check_exist_code(r.status_code): for cur in self._not_exist_flag: if r.text.find(cur)!=-1: return '' return target+';' return '' except Exception as e: return ''
Example #16
Source File: api.py From python-wp with MIT License | 6 votes |
def _get_wp_api_url(self, url): """ Private function for finding the WP-API URL. Arguments --------- url : str WordPress instance URL. """ resp = requests.head(url) # Search the Links for rel="https://api.w.org/". wp_api_rel = resp.links.get('https://api.w.org/') if wp_api_rel: return wp_api_rel['url'] else: # TODO: Rasie a better exception to the rel doesn't exist. raise Exception
Example #17
Source File: s3recon.py From s3recon with MIT License | 6 votes |
def bucket_exists(url, timeout): exists = False public = False try: res = requests.head( url, headers={"User-Agent": choice(useragent_list)}, verify=False, timeout=timeout, ) # TODO: handle redirects status_code = res.status_code exists = status_code != 404 public = status_code == 200 except RequestException: pass return exists, public
Example #18
Source File: util.py From sumgram with MIT License | 5 votes |
def nlpIsServerOn(addr='http://localhost:9000'): try: response = requests.head(addr) if( response.status_code == 200 ): return True else: return False except: genericErrorInfo() return False
Example #19
Source File: tus.py From tus.py with MIT License | 5 votes |
def _get_offset(file_endpoint, headers=None): logger.info("Getting offset") h = {"Tus-Resumable": TUS_VERSION} if headers: h.update(headers) response = requests.head(file_endpoint, headers=h) response.raise_for_status() offset = int(response.headers["Upload-Offset"]) logger.info("offset=%i", offset) return offset
Example #20
Source File: utilities.py From overcast-sonos with BSD 3-Clause "New" or "Revised" License | 5 votes |
def final_redirect_url(url): redirected_url = requests.head(url, allow_redirects=True).url if url != redirected_url: log.debug('''Redirected url '%s' to '%s'.''', url, redirected_url) return redirected_url
Example #21
Source File: bioconductor_skeleton.py From bioconda-utils with MIT License | 5 votes |
def find_best_bioc_version(package, version): """ Given a package version number, identifies which BioC version[s] it is in and returns the latest. If the package is not found, raises a PackageNotFound error. Parameters ---------- package : Case-sensitive Bioconductor package name version : Bioconductor package version Returns None if no valid package found. """ for bioc_version in bioconductor_versions(): for kind, func in zip( ('package', 'data'), ( bioconductor_tarball_url, bioconductor_annotation_data_url, bioconductor_experiment_data_url, ), ): url = func(package, version, bioc_version) if requests.head(url).status_code == 200: logger.debug('success: %s', url) logger.info( 'A working URL for %s==%s was identified for Bioconductor version %s: %s', package, version, bioc_version, url) found_version = bioc_version return found_version else: logger.debug('missing: %s', url) raise PackageNotFoundError( "Cannot find any Bioconductor versions for {0}=={1}".format(package, version) )
Example #22
Source File: check_broken_links.py From psutil with BSD 3-Clause "New" or "Revised" License | 5 votes |
def validate_url(url): """Validate the URL by attempting an HTTP connection. Makes an HTTP-HEAD request for each URL. """ try: res = requests.head(url, timeout=REQUEST_TIMEOUT) # some websites deny 503, like Microsoft # and some send 401, like Apple, observations if (not res.ok) and (res.status_code in RETRY_STATUSES): res = requests.get(url, timeout=REQUEST_TIMEOUT) return res.ok except requests.exceptions.RequestException: return False
Example #23
Source File: StatusCode.py From Python-Scripts with GNU General Public License v3.0 | 5 votes |
def check(out, url): #print("hell") printer("Testing: " + url) link = 'http://' + url try: req = requests.head(link, timeout=10) length = str(req.headers['Content-Length']) scode = str(req.status_code) if scode.startswith("2"): print(blue+"["+bold+green+str(scode)+end+blue+"]"+end+" | "+str(url)+length) elif scode.startswith("3"): if req.headers['Location'].startswith("https://"+url): print(blue+"["+bold+yellow+str(scode)+end+blue+"]"+end+" | "+str(url)+" - HTTPS | "+length) else: print(blue+"["+bold+yellow+str(scode)+end+blue+"]"+end+" | "+str(url)+" | "+req.headers['Location']+" | "+length) elif scode.startswith("4"): print(blue+"["+bold+red+str(scode)+end+blue+"]"+end+" | "+str(url)+" | "+length) else: print(blue+"["+end+str(scode)+blue+"]"+end+" | "+str(url)+" | "+length) if out != 'None': with open(out, 'a') as f: f.write(str(scode)+" | "+url+" | "+length+"\n") f.close() return True except Exception: return False
Example #24
Source File: dirbrute.py From Python-Scripts with GNU General Public License v3.0 | 5 votes |
def checkstatus(domain, url): if url == "" or url == " ": pass elif url.startswith("#"): pass elif len(url) > 30: pass else: printer("Testing: " + domain + url) #time.sleep(1) try: link = domain + url req = requests.head(link) st = str(req.status_code) if st.startswith("2"): print(green + "[+] "+st+" | Found: " + end + "[ " + url + " ]" + " \r") elif st.startswith("3"): link = req.headers['Location'] #link = req.url print(yellow + "[*] "+st+" | Redirection From: " + end + "[ " + url + " ]" + yellow + " -> " + end + "[ " + link + " ]" + " \r") elif st.startswith("1"): print(green + "[+] "+st+" | Found: " + end + "[ " + url + " ]" + " \r") elif st.startswith("4"): if st != '404': print(blue+"[!] "+st+" | Found: " + end + "[ " + url + " ]" + " \r") #writer(link,'up') return True except Exception: #writer(url,'down') return False
Example #25
Source File: tasks.py From sregistry-cli with Mozilla Public License 2.0 | 5 votes |
def download(url, file_name, headers=None, show_progress=True): """stream to a temporary file, rename on successful completion Parameters ========== file_name: the file name to stream to url: the url to stream from headers: additional headers to add """ fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name)) os.close(fd) if DISABLE_SSL_CHECK is True: bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::") verify = not DISABLE_SSL_CHECK # Does the url being requested exist? if requests.head(url, verify=verify).status_code in [200, 401]: response = stream(url, headers=headers, stream_to=tmp_file) if isinstance(response, HTTPError): bot.exit("Error downloading %s, exiting." % url) shutil.move(tmp_file, file_name) else: bot.error("Invalid url or permissions %s" % url) return file_name
Example #26
Source File: http.py From sregistry-cli with Mozilla Public License 2.0 | 5 votes |
def download(self, url, file_name, headers=None, show_progress=True): """stream to a temporary file, rename on successful completion Parameters ========== file_name: the file name to stream to url: the url to stream from headers: additional headers to add force: If the final image exists, don't overwrite show_progress: boolean to show progress bar """ tmp_file = get_tmpfile(prefix="%s.tmp." % file_name) # Should we verify the request? verify = self._verify() # Check here if exists if requests.head(url, verify=verify).status_code in [200, 401]: response = self.stream( url, headers=headers, stream_to=tmp_file, show_progress=show_progress ) if isinstance(response, HTTPError): bot.exit("Error downloading %s, exiting." % url) shutil.move(tmp_file, file_name) else: bot.error("Invalid url or permissions %s" % url) return file_name
Example #27
Source File: bannerscan.py From fuzzdb-collect with GNU General Public License v3.0 | 5 votes |
def run(self): result[self.ip] = dict() for port in PORTS: url_pre = "https://" if port == 443 else "http://" site = url_pre + self.ip + ":" + str(port) try: print ("[*] %s\r" % (site[0:60].ljust(60, " "))), resp = requests.head(site, allow_redirects = False, timeout=self.timeout, headers=self.headers ) result[self.ip][port] = dict() except Exception, e: pass else: result[self.ip][port]["headers"] = resp.headers result[self.ip][port]["available"] = list() for path in PATHS: try: url = site + path print ("[*] %s\r" % (url[0:60].ljust(60, " "))), resp = self.req.get(url, allow_redirects = False, timeout=self.timeout, headers=self.headers ) except Exception, e: pass else: if resp.status_code in [200, 406, 401, 403, 500]: r = re.findall("<title>([\s\S]+?)</title>", resp.content) title = lambda r : r and r[0] or "" result[self.ip][port]["available"].append((title(r), url, resp.status_code))
Example #28
Source File: http.py From sregistry-cli with Mozilla Public License 2.0 | 5 votes |
def head(self, url): """head request, typically used for status code retrieval, etc. """ bot.debug("HEAD %s" % url) return self._call(url, func=requests.head)
Example #29
Source File: bioconductor_skeleton.py From bioconda-utils with MIT License | 5 votes |
def bioarchive_url(self): """ Returns the bioaRchive URL if one exists for this version of this package, otherwise returns None. """ url = bioarchive_url(self.package, self.version, self.bioc_version) response = requests.head(url) if response.status_code == 200: return url
Example #30
Source File: bioconductor_skeleton.py From bioconda-utils with MIT License | 5 votes |
def bioconductor_tarball_url(self): """ Return the url to the tarball from the bioconductor site. """ url = os.path.join(base_url, self.bioc_version, self.packages[self.package]['URLprefix'], self.packages[self.package]['source.ver']) response = requests.head(url) if response.status_code == 200: return url