Python urllib.parse() Examples

The following are code examples for showing how to use urllib.parse(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: drydock   Author: airshipit   File: resolver.py    Apache License 2.0 12 votes vote down vote up
def resolve_reference_http(cls, design_uri):
        """Retrieve design documents from http/https endpoints.

        Return a byte array of the response content. Support unsecured or
        basic auth

        :param design_uri: Tuple as returned by urllib.parse for the design reference
        """
        if design_uri.username is not None and design_uri.password is not None:
            response = requests.get(
                design_uri.geturl(),
                auth=(design_uri.username, design_uri.password),
                timeout=get_client_timeouts())
        else:
            response = requests.get(
                design_uri.geturl(), timeout=get_client_timeouts())

        return response.content 
Example 2
Project: drydock   Author: airshipit   File: resolver.py    Apache License 2.0 6 votes vote down vote up
def resolve_reference_ucp(cls, design_uri):
        """Retrieve artifacts from a Airship service endpoint.

        Return a byte array of the response content. Assumes Keystone
        authentication required.

        :param design_uri: Tuple as returned by urllib.parse for the design reference
        """
        ks_sess = KeystoneUtils.get_session()
        (new_scheme, foo) = re.subn(r'^[^+]+\+', '', design_uri.scheme)
        url = urllib.parse.urlunparse(
            (new_scheme, design_uri.netloc, design_uri.path, design_uri.params,
             design_uri.query, design_uri.fragment))
        LOG.debug("Calling Keystone session for url %s" % str(url))
        resp = ks_sess.get(url, timeout=get_client_timeouts())
        if resp.status_code >= 400:
            raise errors.InvalidDesignReference(
                "Received error code for reference %s: %s - %s" %
                (url, str(resp.status_code), resp.text))
        return resp.content 
Example 3
Project: colorcore   Author: martexcoin   File: routing.py    MIT License 6 votes vote down vote up
def execute():
        parser = configparser.ConfigParser()
        parser.read('config.ini')

        configuration = Configuration(parser)

        class NetworkParams(bitcoin.core.CoreChainParams):
            BASE58_PREFIXES = {'PUBKEY_ADDR':configuration.version_byte, 'SCRIPT_ADDR':configuration.p2sh_byte}

        bitcoin.params = NetworkParams()
        router = Router(
            colorcore.operations.Controller,
            sys.stdout,
            lambda: colorcore.caching.SqliteCache(configuration.cache_path),
            configuration,
            asyncio.new_event_loop(),
            "Colorcore: The Open Assets client for colored coins")
        router.parse(sys.argv[1:]) 
Example 4
Project: jawfish   Author: war-and-code   File: xmlbuilder.py    MIT License 6 votes vote down vote up
def resolveEntity(self, publicId, systemId):
        assert systemId is not None
        source = DOMInputSource()
        source.publicId = publicId
        source.systemId = systemId
        source.byteStream = self._get_opener().open(systemId)

        # determine the encoding if the transport provided it
        source.encoding = self._guess_media_encoding(source)

        # determine the base URI is we can
        import posixpath, urllib.parse
        parts = urllib.parse.urlparse(systemId)
        scheme, netloc, path, params, query, fragment = parts
        # XXX should we check the scheme here as well?
        if path and not path.endswith("/"):
            path = posixpath.dirname(path) + "/"
            parts = scheme, netloc, path, params, query, fragment
            source.baseURI = urllib.parse.urlunparse(parts)

        return source 
Example 5
Project: smoke-zephyr   Author: zeroSteiner   File: utilities.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def download(url, filename=None):
	"""
	Download a file from a url and save it to disk.

	:param str url: The URL to fetch the file from.
	:param str filename: The destination file to write the data to.
	"""
	# requirements os, shutil, urllib.parse, urllib.request
	if not filename:
		url_parts = urllib.parse.urlparse(url)
		filename = os.path.basename(url_parts.path)
	url_h = urllib.request.urlopen(url)
	with open(filename, 'wb') as file_h:
		shutil.copyfileobj(url_h, file_h)
	url_h.close()
	return 
Example 6
Project: thewarden   Author: pxsocs   File: pricing.py    MIT License 6 votes vote down vote up
def __init__(self,
                 name,
                 base_url,
                 ticker_field,
                 field_dict=None,
                 doc_link=None):
        # field dict includes all fields to be passed to the URL
        # for example, for Alphavantage
        # name = 'Alphavantage_digital'
        # base-url = 'https://www.alphavantage.co/query'
        # ticker_field = 'symbol'
        # field_dict = {'function': 'DIGITAL_CURRENCY_DAILY',
        #               'market': 'CNY',
        #               'apikey': 'demo')
        # doc_link = 'https://www.alphavantage.co/documentation/'
        # parse_dict = {'open' : '1a. open (USD)', ...}
        self.name = name.lower()
        self.base_url = base_url
        self.ticker_field = ticker_field
        self.field_dict = field_dict
        self.doc_link = doc_link
        if self.field_dict is not None:
            self.url_args = "&" + urllib.parse.urlencode(field_dict)
        self.errors = [] 
Example 7
Project: WebMap   Author: SECUREFOREST   File: views.py    GNU General Public License v3.0 6 votes vote down vote up
def scan_diff(request, f1, f2):
	r = {}

	if 'auth' not in request.session:
		return render(request, 'nmapreport/nmap_auth.html', r)
	else:
		r['auth'] = True

	try:
		if xmltodict.parse(open('/opt/xml/'+f1, 'r').read()) is not None:
			r['f1'] = f1
		if xmltodict.parse(open('/opt/xml/'+f2, 'r').read()) is not None:
			r['f2'] = f2
	except:
		r['f1'] = ''
		r['f2'] = ''

	return render(request, 'nmapreport/nmap_ndiff.html', r) 
Example 8
Project: ssrspeed_backup   Author: mazhenting   File: v2ray_parser.py    GNU General Public License v3.0 6 votes vote down vote up
def readGuiConfig(self,filename):
		pv2rc = ParserV2RayClash()
		v2rnp = ParserV2RayN()
		rawGuiConfigs = v2rnp.parseGuiConfig(filename)
		if (rawGuiConfigs == False):
			logger.info("Not V2RayN Config.")
			rawGuiConfigs = pv2rc.parseGuiConfig(filename)
			if (rawGuiConfigs == False):
				logger.info("Not Clash Config.")
				logger.critical("Gui config parse failed.")
				rawGuiConfigs = []

		for _dict in rawGuiConfigs:
			_cfg = self.__generateConfig(_dict)
		#	logger.debug(_cfg)
			self._configList.append(_cfg)
		logger.info("Read %d node(s)" % len(self._configList))
	#	logger.critical("V2RayN configuration file will be support soon.") 
Example 9
Project: NiujiaoDebugger   Author: MrSrc   File: utils.py    GNU General Public License v3.0 6 votes vote down vote up
def collapse_rfc2231_value(value, errors='replace',
                           fallback_charset='us-ascii'):
    if not isinstance(value, tuple) or len(value) != 3:
        return unquote(value)
    # While value comes to us as a unicode string, we need it to be a bytes
    # object.  We do not want bytes() normal utf-8 decoder, we want a straight
    # interpretation of the string as character bytes.
    charset, language, text = value
    if charset is None:
        # Issue 17369: if charset/lang is None, decode_rfc2231 couldn't parse
        # the value, so use the fallback_charset.
        charset = fallback_charset
    rawbytes = bytes(text, 'raw-unicode-escape')
    try:
        return str(rawbytes, charset, errors)
    except LookupError:
        # charset is not a known codec.
        return unquote(text)


#
# datetime doesn't provide a localtime function yet, so provide one.  Code
# adapted from the patch in issue 9527.  This may not be perfect, but it is
# better than not having it.
# 
Example 10
Project: NiujiaoDebugger   Author: MrSrc   File: xmlbuilder.py    GNU General Public License v3.0 6 votes vote down vote up
def resolveEntity(self, publicId, systemId):
        assert systemId is not None
        source = DOMInputSource()
        source.publicId = publicId
        source.systemId = systemId
        source.byteStream = self._get_opener().open(systemId)

        # determine the encoding if the transport provided it
        source.encoding = self._guess_media_encoding(source)

        # determine the base URI is we can
        import posixpath, urllib.parse
        parts = urllib.parse.urlparse(systemId)
        scheme, netloc, path, params, query, fragment = parts
        # XXX should we check the scheme here as well?
        if path and not path.endswith("/"):
            path = posixpath.dirname(path) + "/"
            parts = scheme, netloc, path, params, query, fragment
            source.baseURI = urllib.parse.urlunparse(parts)

        return source 
Example 11
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 6 votes vote down vote up
def test_urldefrag(self):
        str_cases = [
            ('http://python.org#frag', 'http://python.org', 'frag'),
            ('http://python.org', 'http://python.org', ''),
            ('http://python.org/#frag', 'http://python.org/', 'frag'),
            ('http://python.org/', 'http://python.org/', ''),
            ('http://python.org/?q#frag', 'http://python.org/?q', 'frag'),
            ('http://python.org/?q', 'http://python.org/?q', ''),
            ('http://python.org/p#frag', 'http://python.org/p', 'frag'),
            ('http://python.org/p?q', 'http://python.org/p?q', ''),
            (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'),
            (RFC2396_BASE, 'http://a/b/c/d;p?q', ''),
        ]
        def _encode(t):
            return type(t)(x.encode('ascii') for x in t)
        bytes_cases = [_encode(x) for x in str_cases]
        for url, defrag, frag in str_cases + bytes_cases:
            result = urllib.parse.urldefrag(url)
            self.assertEqual(result.geturl(), url)
            self.assertEqual(result, (defrag, frag))
            self.assertEqual(result.url, defrag)
            self.assertEqual(result.fragment, frag) 
Example 12
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 6 votes vote down vote up
def test_withoutscheme(self):
        # Test urlparse without scheme
        # Issue 754016: urlparse goes wrong with IP:port without scheme
        # RFC 1808 specifies that netloc should start with //, urlparse expects
        # the same, otherwise it classifies the portion of url as path.
        self.assertEqual(urllib.parse.urlparse("path"),
                ('','','path','','',''))
        self.assertEqual(urllib.parse.urlparse("//www.python.org:80"),
                ('','www.python.org:80','','','',''))
        self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
                ('http','www.python.org:80','','','',''))
        # Repeat for bytes input
        self.assertEqual(urllib.parse.urlparse(b"path"),
                (b'',b'',b'path',b'',b'',b''))
        self.assertEqual(urllib.parse.urlparse(b"//www.python.org:80"),
                (b'',b'www.python.org:80',b'',b'',b'',b''))
        self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
                (b'http',b'www.python.org:80',b'',b'',b'',b'')) 
Example 13
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 6 votes vote down vote up
def test_mixed_types_rejected(self):
        # Several functions that process either strings or ASCII encoded bytes
        # accept multiple arguments. Check they reject mixed type input
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlparse("www.python.org", b"http")
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlparse(b"www.python.org", "http")
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlsplit("www.python.org", b"http")
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlsplit(b"www.python.org", "http")
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlunparse(( b"http", "www.python.org","","","",""))
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlunparse(("http", b"www.python.org","","","",""))
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlunsplit((b"http", "www.python.org","","",""))
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urlunsplit(("http", b"www.python.org","","",""))
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urljoin("http://python.org", b"http://python.org")
        with self.assertRaisesRegex(TypeError, "Cannot mix str"):
            urllib.parse.urljoin(b"http://python.org", "http://python.org") 
Example 14
Project: fs_image   Author: facebookincubator   File: repo_downloader.py    MIT License 5 votes vote down vote up
def _download(self, relative_url):
        assert not relative_url.startswith('/')
        with open_url(
            urllib.parse.urljoin(self._repo_url, relative_url)
        ) as input:
            yield input

    # May raise `ReportableError`s to be caught by `_download_repodatas` 
Example 15
Project: xadmin_bugfix   Author: vip68   File: themes.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _get_theme(self):
        if self.user:
            try:
                return UserSettings.objects.get(user=self.user, key="site-theme").value
            except Exception:
                pass
        if '_theme' in self.request.COOKIES:
            if six.PY2:
                func = urllib.unquote
            else:
                func = urllib.parse.unquote
            return func(self.request.COOKIES['_theme'])
        return self.default_theme 
Example 16
Project: drydock   Author: airshipit   File: resolver.py    Apache License 2.0 5 votes vote down vote up
def resolve_reference(cls, design_ref):
        """Resolve a reference to a design document.

        Locate a schema handler based on the URI scheme of the data reference
        and use that handler to get the data referenced.

        :param design_ref: A URI-formatted reference to a data entity
        """
        try:
            design_uri = urllib.parse.urlparse(design_ref)

            handler = cls.scheme_handlers.get(design_uri.scheme, None)

            if handler is None:
                raise errors.InvalidDesignReference(
                    "Invalid reference scheme %s: no handler." %
                    design_uri.scheme)
            else:
                tries = 0
                while tries < config_mgr.conf.network.http_client_retries:
                    try:
                        # Have to do a little magic to call the classmethod as a pointer
                        return handler.__get__(None, cls)(design_uri)
                    except Exception as ex:
                        tries = tries + 1
                        if tries < config_mgr.conf.network.http_client_retries:
                            LOG.debug("Retrying reference after failure: %s" %
                                      str(ex))
                            time.sleep(5**tries)
        except ValueError:
            raise errors.InvalidDesignReference(
                "Cannot resolve design reference %s: unable to parse as valid URI."
                % design_ref) 
Example 17
Project: drydock   Author: airshipit   File: resolver.py    Apache License 2.0 5 votes vote down vote up
def resolve_reference_file(cls, design_uri):
        """Retrieve design documents from local file endpoints.

        Return a byte array of the file contents

        :param design_uri: Tuple as returned by urllib.parse for the design reference
        """
        if design_uri.path != '':
            f = open(design_uri.path, 'rb')
            doc = f.read()
            return doc 
Example 18
Project: django-xadmin   Author: MarkHoo   File: themes.py    MIT License 5 votes vote down vote up
def _get_theme(self):
        if self.user:
            try:
                return UserSettings.objects.get(user=self.user, key="site-theme").value
            except Exception:
                pass
        if '_theme' in self.request.COOKIES:
            if six.PY2:
                func = urllib.unquote
            else:
                func = urllib.parse.unquote
            return func(self.request.COOKIES['_theme'])
        return self.default_theme 
Example 19
Project: django-xadmin   Author: MarkHoo   File: themes.py    MIT License 5 votes vote down vote up
def _get_theme(self):
        if self.user:
            try:
                return UserSettings.objects.get(user=self.user, key="site-theme").value
            except Exception:
                pass
        if '_theme' in self.request.COOKIES:
            if six.PY2:
                func = urllib.unquote
            else:
                func = urllib.parse.unquote
            return func(self.request.COOKIES['_theme'])
        return self.default_theme 
Example 20
Project: django-xadmin   Author: MarkHoo   File: themes.py    MIT License 5 votes vote down vote up
def _get_theme(self):
        if self.user:
            try:
                return UserSettings.objects.get(user=self.user, key="site-theme").value
            except Exception:
                pass
        if '_theme' in self.request.COOKIES:
            if six.PY2:
                func = urllib.unquote
            else:
                func = urllib.parse.unquote
            return func(self.request.COOKIES['_theme'])
        return self.default_theme 
Example 21
Project: grove   Author: jaredthecoder   File: LoginResource.py    MIT License 5 votes vote down vote up
def get_canonical_facebook_profile_pic_url(self, user_id):
        resp = requests.get('http://graph.facebook.com/' +
                            '{user_id}/picture?type=large'.format(
                                user_id=user_id))
        return urllib.parse.quote(resp.url) 
Example 22
Project: sic   Author: Yanixos   File: bazaar.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, url=None, *args, **kwargs):
        super(Bazaar, self).__init__(url, *args, **kwargs)
        # Python >= 2.7.4, 3.3 doesn't have uses_fragment or non_hierarchical
        # Register lp but do not expose as a scheme to support bzr+lp.
        if getattr(urllib_parse, 'uses_fragment', None):
            urllib_parse.uses_fragment.extend(['lp'])
            urllib_parse.non_hierarchical.extend(['lp']) 
Example 23
Project: AshsSDK   Author: thehappydinoa   File: bazaar.py    MIT License 5 votes vote down vote up
def __init__(self, url=None, *args, **kwargs):
        super(Bazaar, self).__init__(url, *args, **kwargs)
        # Python >= 2.7.4, 3.3 doesn't have uses_fragment or non_hierarchical
        # Register lp but do not expose as a scheme to support bzr+lp.
        if getattr(urllib_parse, 'uses_fragment', None):
            urllib_parse.uses_fragment.extend(['lp'])
            urllib_parse.non_hierarchical.extend(['lp']) 
Example 24
Project: subtake   Author: kp625544   File: sublist3r.py    GNU General Public License v2.0 5 votes vote down vote up
def parse_args():
    # parse the arguments
    parser = argparse.ArgumentParser(epilog='\tExample: \r\npython ' + sys.argv[0] + " -d google.com")
    parser.error = parser_error
    parser._optionals.title = "OPTIONS"
    parser.add_argument('-d', '--domain', help="Domain name to enumerate it's subdomains", required=True)
    parser.add_argument('-b', '--bruteforce', help='Enable the subbrute bruteforce module', nargs='?', default=False)
    parser.add_argument('-p', '--ports', help='Scan the found subdomains against specified tcp ports')
    parser.add_argument('-v', '--verbose', help='Enable Verbosity and display results in realtime', nargs='?', default=False)
    parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=30)
    parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines')
    parser.add_argument('-o', '--output', help='Save the results to text file')
    return parser.parse_args() 
Example 25
Project: PheKnowLator   Author: callahantiff   File: EdgeDictionary.py    Apache License 2.0 5 votes vote down vote up
def queries_uniprot_api(data_file):
        """Searches a list of entities against the Uniprot API (uniprot.org/help/api_idmapping).

        Args:
            data_file (str): A filepath containing data to map.

        Returns:
            A dictionary of results returned from mapping identifiers.

        Raises:
            An exception is raised if the generated dictionary does not have the same number of rows as the what was
            returned by the API.
        """
        proteins = list(set([x.split('\\t')[1] for x in open(data_file).read().split('!')[-1].split('\\n')[1:]]))
        params = {'from': 'ACC+ID', 'to': 'P_ENTREZGENEID', 'format': 'tab', 'query': ' '.join(proteins)}

        data = urllib.parse.urlencode(params).encode('utf-8')
        requests = urllib.request.Request('https://www.uniprot.org/uploadlists/', data)
        results = urllib.request.urlopen(requests).read().decode('utf-8')

        # convert results to dictionary
        api_results = {}
        for res in results.split('\n')[1:-1]:
            res_row = res.split('\t')
            if str(res_row[0]) in api_results.keys():
                api_results[str(res_row[0])].append('http://purl.uniprot.org/geneid/' + str(res_row[1]))

            else:
                api_results[str(res_row[0])] = ['http://purl.uniprot.org/geneid/' + str(res_row[1])]

        # CHECK - all URLs returned an data file
        if len(api_results) == 0:
            raise Exception('ERROR: API returned no data')
        else:
            return api_results 
Example 26
Project: colorcore   Author: martexcoin   File: routing.py    MIT License 5 votes vote down vote up
def parse(self, args):
        """
        Parses the arguments and executes the corresponding operation.

        :param list[str] args: The arguments to parse.
        """
        args = vars(self._parser.parse_args(args))
        func = args.pop('_func', self._parser.print_usage)
        func(**args) 
Example 27
Project: linjuanbang   Author: fengjinqi   File: base.py    MIT License 5 votes vote down vote up
def getResponse(self, accessToken=None, version='2.0', timeout=30):
        # =======================================================================
        # 获取response结果
        # =======================================================================
        sys_parameters = {
            P_FORMAT: 'json',
            P_APPKEY: self.__app_key,
            P_SIGN_METHOD: "md5",
            P_VERSION: '1.0',
            P_TIMESTAMP: str(self.getTime()),
            #P_PARTNER_ID: SYSTEM_GENERATE_VERSION,
            P_API: self.getapiname(),
        }
        if accessToken is not None:
            sys_parameters[P_ACCESS_TOKEN] = accessToken
        application_parameter = self.getApplicationParameters()

        sys_parameters[P_JSON_PARAM_KEY] = json.dumps(application_parameter, ensure_ascii=False,
                                                      default=lambda value: value.__dict__)
        sys_parameters[P_SIGN] = sign(self.__secret, sys_parameters)
        from urllib.parse import urlencode

        #print(self.__domain+"?"+urlencode(sys_parameters))
        #result = requests.get(self.__domain+"?"+urlencode(sys_parameters))

        header = self.get_request_header()
        result = requests.request(self.__httpmethod,self.__domain,data=sys_parameters,headers=header,timeout=timeout)

        jsonobj = {}
        try:
            jsonobj = result.json()
        except Exception as e:
            jsonobj =e
        return jsonobj 
Example 28
Project: jawfish   Author: war-and-code   File: xmlbuilder.py    MIT License 5 votes vote down vote up
def parseURI(self, uri):
        if self.entityResolver:
            input = self.entityResolver.resolveEntity(None, uri)
        else:
            input = DOMEntityResolver().resolveEntity(None, uri)
        return self.parse(input) 
Example 29
Project: jawfish   Author: war-and-code   File: xmlbuilder.py    MIT License 5 votes vote down vote up
def parse(self, input):
        options = copy.copy(self._options)
        options.filter = self.filter
        options.errorHandler = self.errorHandler
        fp = input.byteStream
        if fp is None and options.systemId:
            import urllib.request
            fp = urllib.request.urlopen(input.systemId)
        return self._parse_bytestream(fp, options) 
Example 30
Project: jawfish   Author: war-and-code   File: saxutils.py    MIT License 5 votes vote down vote up
def parse(self, source):
        self._parent.setContentHandler(self)
        self._parent.setErrorHandler(self)
        self._parent.setEntityResolver(self)
        self._parent.setDTDHandler(self)
        self._parent.parse(source) 
Example 31
Project: jawfish   Author: war-and-code   File: saxutils.py    MIT License 5 votes vote down vote up
def prepare_input_source(source, base=""):
    """This function takes an InputSource and an optional base URL and
    returns a fully resolved InputSource object ready for reading."""

    if isinstance(source, str):
        source = xmlreader.InputSource(source)
    elif hasattr(source, "read"):
        f = source
        source = xmlreader.InputSource()
        source.setByteStream(f)
        if hasattr(f, "name"):
            source.setSystemId(f.name)

    if source.getByteStream() is None:
        sysid = source.getSystemId()
        basehead = os.path.dirname(os.path.normpath(base))
        sysidfilename = os.path.join(basehead, sysid)
        if os.path.isfile(sysidfilename):
            source.setSystemId(sysidfilename)
            f = open(sysidfilename, "rb")
        else:
            source.setSystemId(urllib.parse.urljoin(base, sysid))
            f = urllib.request.urlopen(source.getSystemId())

        source.setByteStream(f)

    return source 
Example 32
Project: steam_workshop_downloader   Author: Geam   File: workshop.py    MIT License 5 votes vote down vote up
def get_plugins_info (plugins_id_list):
    """Ask api the info on each plugin(s)
    Will return:
        - error:
            - None if no error encounter
            - the error an error occur
        - an array of plugin with the all the data return by the steam api
    Return error(error), array(array_of_plugins)
    """
    json_response = []
    error = None
    data = const_data['file']
    data['itemcount'] = len(plugins_id_list)
    for idx, plugin_id in enumerate(plugins_id_list):
        data['publishedfileids[' + str(idx) + ']'] = plugin_id
    encode_data = urllib.parse.urlencode(data).encode('ascii')
    try:
        response = urllib.request.urlopen(const_urls['file'], encode_data)
    except HTTPError as e:
        print("Server return " + str(e.code) + " error")
        error = e
    except URLError as e:
        print("Can't reach server: " + e.reason)
        error = e
    else:
        json_response = json.loads(response.read().decode('utf8'))
        json_response = json_response['response']['publishedfiledetails']
    return error, json_response 
Example 33
Project: smoke-zephyr   Author: zeroSteiner   File: utilities.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def parse_to_slug(words, maxlen=24):
	"""
	Parse a string into a slug format suitable for use in URLs and other
	character restricted applications. Only utf-8 strings are supported at this
	time.

	:param str words: The words to parse.
	:param int maxlen: The maximum length of the slug.
	:return: The parsed words as a slug.
	:rtype: str
	"""
	slug = ''
	maxlen = min(maxlen, len(words))
	for c in words:
		if len(slug) == maxlen:
			break
		c = ord(c)
		if c == 0x27:
			continue
		elif c >= 0x30 and c <= 0x39:
			slug += chr(c)
		elif c >= 0x41 and c <= 0x5a:
			slug += chr(c + 0x20)
		elif c >= 0x61 and c <= 0x7a:
			slug += chr(c)
		elif len(slug) and slug[-1] != '-':
			slug += '-'
	if len(slug) and slug[-1] == '-':
		slug = slug[:-1]
	return slug 
Example 34
Project: Repobot   Author: Desgard   File: bazaar.py    MIT License 5 votes vote down vote up
def __init__(self, url=None, *args, **kwargs):
        super(Bazaar, self).__init__(url, *args, **kwargs)
        # Python >= 2.7.4, 3.3 doesn't have uses_fragment or non_hierarchical
        # Register lp but do not expose as a scheme to support bzr+lp.
        if getattr(urllib_parse, 'uses_fragment', None):
            urllib_parse.uses_fragment.extend(['lp'])
            urllib_parse.non_hierarchical.extend(['lp']) 
Example 35
Project: thewarden   Author: pxsocs   File: pricing.py    MIT License 5 votes vote down vote up
def request_data(self, ticker):
        data = None
        if self.base_url is not None:
            ticker = ticker.upper()
            globalURL = (self.base_url + "?" + self.ticker_field + "=" +
                         ticker + self.url_args)
            # Some APIs use the ticker without a ticker field i.e. xx.xx./AAPL&...
            # in these cases, we pass the ticker field as empty
            if self.ticker_field == '':
                if self.url_args[0] == '&':
                    self.url_args = self.url_args.replace('&', '?', 1)
                globalURL = (self.base_url + "/" + ticker + self.url_args)
            request = tor_request(globalURL)
            try:
                data = request.json()
            except Exception:
                try:  # Try again - some APIs return a json already
                    data = json.loads(request)
                except Exception as e:
                    self.errors.append(e)
        return (data)


# PriceData Class Information
# Example on how to create a ticker class (PriceData)
# provider = PROVIDER_LIST['cc_digital']
# btc = PriceData("BTC", provider)
# btc.errors:       Any error messages
# btc.provider:     Provider being used for requests
# btc.filename:     Local filename where historical prices are saved
# Other info:
# btc.ticker, btc.last_update, btc.first_update, btc.last_close
# btc.update_history(force=False)
# btc.df_fx(currency, fx_provider): returns a df with
#                                   prices and fx conversions
# btc.price_ondate(date)
# btc.price_parser(): do not use directly. This is used to parse
#                     the requested data from the API provider
# btc.realtime(provider): returns realtime price (float) 
Example 36
Project: WebMap   Author: SECUREFOREST   File: api.py    GNU General Public License v3.0 5 votes vote down vote up
def port_details(request, address, portid):
	r = {}

	if 'auth' not in request.session:
		return False

	oo = xmltodict.parse(open('/opt/xml/'+request.session['scanfile'], 'r').read())
	r['out'] = json.dumps(oo['nmaprun'], indent=4)
	o = json.loads(r['out'])

	for ik in o['host']:

		# this fix single host report
		if type(ik) is dict:
			i = ik
		else:
			i = o['host']

		if '@addr' in i['address']:
			saddress = i['address']['@addr']
		elif type(i['address']) is list:
			for ai in i['address']:
				if ai['@addrtype'] == 'ipv4':
					saddress = ai['@addr'] 

		if str(saddress) == address:
			for pobj in i['ports']['port']:
				if type(pobj) is dict:
					p = pobj
				else:
					p = i['ports']['port']

				if p['@portid'] == portid:
					return HttpResponse(json.dumps(p, indent=4), content_type="application/json") 
Example 37
Project: WebMap   Author: SECUREFOREST   File: api.py    GNU General Public License v3.0 5 votes vote down vote up
def apiv1_scan(request):
	r = {}
	if token_check(request.GET['token']) is not True:
		return HttpResponse(json.dumps({'error':'invalid token'}, indent=4), content_type="application/json")

	gitcmd = os.popen('cd /opt/nmapdashboard/nmapreport && git rev-parse --abbrev-ref HEAD')
	r['webmap_version'] = gitcmd.read().strip()

	xmlfiles = os.listdir('/opt/xml')

	r['scans'] = {}

	xmlfilescount = 0
	for i in xmlfiles:
		if re.search('\.xml$', i) is None:
			continue

		xmlfilescount = (xmlfilescount + 1)

		try:
			oo = xmltodict.parse(open('/opt/xml/'+i, 'r').read())
		except:
			r['scans'][i] = {'filename':html.escape(i), 'startstr': '', 'nhost':0, 'port_stats':{'open':0,'closed':0,'filtered':0}}
			continue

		rout = json.dumps(oo['nmaprun'], indent=4)
		o = json.loads(rout)

		if 'host' in o:
			if type(o['host']) is not dict:
				hostnum = str(len(o['host']))
			else:
				hostnum = '1'
		else:
			hostnum = '0'

		portstats = nmap_ports_stats(i)

		r['scans'][i] = {'filename':html.escape(i), 'startstr': html.escape(o['@startstr']), 'nhost':hostnum, 'port_stats':{'open':portstats['po'],'closed':portstats['pc'],'filtered':portstats['pf']}}

	return HttpResponse(json.dumps(r, indent=4), content_type="application/json") 
Example 38
Project: RTX   Author: RTXteam   File: multi_owl_to_json_kg.py    MIT License 5 votes vote down vote up
def is_ignorable_ontology_term(iri: str):
    parsed_iri = urllib.parse.urlparse(iri)
    iri_netloc = parsed_iri.netloc
    iri_path = parsed_iri.path
    return iri_netloc in ('example.com', 'usefulinc.com') or iri_path.startswith('/ontology/provisional') 
Example 39
Project: chattR   Author: patrickstocklin   File: bazaar.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, url=None, *args, **kwargs):
        super(Bazaar, self).__init__(url, *args, **kwargs)
        # Python >= 2.7.4, 3.3 doesn't have uses_fragment or non_hierarchical
        # Register lp but do not expose as a scheme to support bzr+lp.
        if getattr(urllib_parse, 'uses_fragment', None):
            urllib_parse.uses_fragment.extend(['lp'])
            urllib_parse.non_hierarchical.extend(['lp']) 
Example 40
Project: Python_Master_Courses   Author: makelove   File: 域名过滤1.py    GNU General Public License v3.0 5 votes vote down vote up
def domain_filter(url, filter_list):
    try:
        pas: urllib.parse.ParseResult = urlparse(url)
        main_domain = '.'.join(pas.hostname.split('.')[-2:])
        return main_domain in filter_list
    except Exception as e:
        print(traceback.format_exc())
        return False 
Example 41
Project: NiujiaoDebugger   Author: MrSrc   File: utils.py    GNU General Public License v3.0 5 votes vote down vote up
def parseaddr(addr):
    """
    Parse addr into its constituent realname and email address parts.

    Return a tuple of realname and email address, unless the parse fails, in
    which case return a 2-tuple of ('', '').
    """
    addrs = _AddressList(addr).addresslist
    if not addrs:
        return '', ''
    return addrs[0]


# rfc822.unquote() doesn't properly de-backslash-ify in Python pre-2.3. 
Example 42
Project: NiujiaoDebugger   Author: MrSrc   File: utils.py    GNU General Public License v3.0 5 votes vote down vote up
def encode_rfc2231(s, charset=None, language=None):
    """Encode string according to RFC 2231.

    If neither charset nor language is given, then s is returned as-is.  If
    charset is given but not language, the string is encoded using the empty
    string for language.
    """
    s = urllib.parse.quote(s, safe='', encoding=charset or 'ascii')
    if charset is None and language is None:
        return s
    if language is None:
        language = ''
    return "%s'%s'%s" % (charset, language, s) 
Example 43
Project: NiujiaoDebugger   Author: MrSrc   File: cgi.py    GNU General Public License v3.0 5 votes vote down vote up
def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
    """Parse a query given as a string argument."""
    warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead",
         DeprecationWarning, 2)
    return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing) 
Example 44
Project: NiujiaoDebugger   Author: MrSrc   File: cgi.py    GNU General Public License v3.0 5 votes vote down vote up
def read_urlencoded(self):
        """Internal: read data in query string format."""
        qs = self.fp.read(self.length)
        if not isinstance(qs, bytes):
            raise ValueError("%s should return bytes, got %s" \
                             % (self.fp, type(qs).__name__))
        qs = qs.decode(self.encoding, self.errors)
        if self.qs_on_post:
            qs += '&' + self.qs_on_post
        query = urllib.parse.parse_qsl(
            qs, self.keep_blank_values, self.strict_parsing,
            encoding=self.encoding, errors=self.errors,
            max_num_fields=self.max_num_fields)
        self.list = [MiniFieldStorage(key, value) for key, value in query]
        self.skip_lines() 
Example 45
Project: NiujiaoDebugger   Author: MrSrc   File: server.py    GNU General Public License v3.0 5 votes vote down vote up
def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        # abandon query parameters
        path = path.split('?',1)[0]
        path = path.split('#',1)[0]
        # Don't forget explicit trailing slash when normalizing. Issue17324
        trailing_slash = path.rstrip().endswith('/')
        try:
            path = urllib.parse.unquote(path, errors='surrogatepass')
        except UnicodeDecodeError:
            path = urllib.parse.unquote(path)
        path = posixpath.normpath(path)
        words = path.split('/')
        words = filter(None, words)
        path = self.directory
        for word in words:
            if os.path.dirname(word) or word in (os.curdir, os.pardir):
                # Ignore components that are not a simple file/directory name
                continue
            path = os.path.join(path, word)
        if trailing_slash:
            path += '/'
        return path 
Example 46
Project: NiujiaoDebugger   Author: MrSrc   File: cookiejar.py    GNU General Public License v3.0 5 votes vote down vote up
def iso2time(text):
    """
    As for http2time, but parses the ISO 8601 formats:

    1994-02-03 14:15:29 -0100    -- ISO 8601 format
    1994-02-03 14:15:29          -- zone is optional
    1994-02-03                   -- only date
    1994-02-03T14:15:29          -- Use T as separator
    19940203T141529Z             -- ISO 8601 compact format
    19940203                     -- only date

    """
    # clean up
    text = text.lstrip()

    # tz is time zone specifier string
    day, mon, yr, hr, min, sec, tz = [None]*7

    # loose regexp parse
    m = ISO_DATE_RE.search(text)
    if m is not None:
        # XXX there's an extra bit of the timezone I'm ignoring here: is
        #   this the right thing to do?
        yr, mon, day, hr, min, sec, tz, _ = m.groups()
    else:
        return None  # bad format

    return _str2time(day, mon, yr, hr, min, sec, tz)


# Header parsing
# ----------------------------------------------------------------------------- 
Example 47
Project: NiujiaoDebugger   Author: MrSrc   File: cookiejar.py    GNU General Public License v3.0 5 votes vote down vote up
def request_host(request):
    """Return request-host, as defined by RFC 2965.

    Variation from RFC: returned value is lowercased, for convenient
    comparison.

    """
    url = request.get_full_url()
    host = urllib.parse.urlparse(url)[1]
    if host == "":
        host = request.get_header("Host", "")

    # remove port, if present
    host = cut_port_re.sub("", host, 1)
    return host.lower() 
Example 48
Project: NiujiaoDebugger   Author: MrSrc   File: cookiejar.py    GNU General Public License v3.0 5 votes vote down vote up
def request_path(request):
    """Path component of request-URI, as defined by RFC 2965."""
    url = request.get_full_url()
    parts = urllib.parse.urlsplit(url)
    path = escape_path(parts.path)
    if not path.startswith("/"):
        # fix bad RFC 2396 absoluteURI
        path = "/" + path
    return path 
Example 49
Project: NiujiaoDebugger   Author: MrSrc   File: cookiejar.py    GNU General Public License v3.0 5 votes vote down vote up
def escape_path(path):
    """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
    # There's no knowing what character encoding was used to create URLs
    # containing %-escapes, but since we have to pick one to escape invalid
    # path characters, we pick UTF-8, as recommended in the HTML 4.0
    # specification:
    # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
    # And here, kind of: draft-fielding-uri-rfc2396bis-03
    # (And in draft IRI specification: draft-duerst-iri-05)
    # (And here, for new URI schemes: RFC 2718)
    path = urllib.parse.quote(path, HTTP_PATH_SAFE)
    path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
    return path 
Example 50
Project: NiujiaoDebugger   Author: MrSrc   File: client.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, use_datetime=False, use_builtin_types=False):
        self._use_datetime = use_datetime
        self._use_builtin_types = use_builtin_types
        self._connection = (None, None)
        self._extra_headers = []

    ##
    # Send a complete request, and parse the response.
    # Retry request if a cached connection has disconnected.
    #
    # @param host Target host.
    # @param handler Target PRC handler.
    # @param request_body XML-RPC request body.
    # @param verbose Debugging flag.
    # @return Parsed response. 
Example 51
Project: NiujiaoDebugger   Author: MrSrc   File: client.py    GNU General Public License v3.0 5 votes vote down vote up
def parse_response(self, response):
        # read response data from httpresponse, and parse it
        # Check for new http response object, otherwise it is a file object.
        if hasattr(response, 'getheader'):
            if response.getheader("Content-Encoding", "") == "gzip":
                stream = GzipDecodedResponse(response)
            else:
                stream = response
        else:
            stream = response

        p, u = self.getparser()

        while 1:
            data = stream.read(1024)
            if not data:
                break
            if self.verbose:
                print("body:", repr(data))
            p.feed(data)

        if stream is not response:
            stream.close()
        p.close()

        return u.close()

##
# Standard transport class for XML-RPC over HTTPS. 
Example 52
Project: NiujiaoDebugger   Author: MrSrc   File: client.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=False,
                 allow_none=False, use_datetime=False, use_builtin_types=False,
                 *, context=None):
        # establish a "logical" server connection

        # get the url
        type, uri = urllib.parse.splittype(uri)
        if type not in ("http", "https"):
            raise OSError("unsupported XML-RPC protocol")
        self.__host, self.__handler = urllib.parse.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                handler = SafeTransport
                extra_kwargs = {"context": context}
            else:
                handler = Transport
                extra_kwargs = {}
            transport = handler(use_datetime=use_datetime,
                                use_builtin_types=use_builtin_types,
                                **extra_kwargs)
        self.__transport = transport

        self.__encoding = encoding or 'utf-8'
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 53
Project: NiujiaoDebugger   Author: MrSrc   File: xmlbuilder.py    GNU General Public License v3.0 5 votes vote down vote up
def parseURI(self, uri):
        if self.entityResolver:
            input = self.entityResolver.resolveEntity(None, uri)
        else:
            input = DOMEntityResolver().resolveEntity(None, uri)
        return self.parse(input) 
Example 54
Project: NiujiaoDebugger   Author: MrSrc   File: xmlbuilder.py    GNU General Public License v3.0 5 votes vote down vote up
def parse(self, input):
        options = copy.copy(self._options)
        options.filter = self.filter
        options.errorHandler = self.errorHandler
        fp = input.byteStream
        if fp is None and options.systemId:
            import urllib.request
            fp = urllib.request.urlopen(input.systemId)
        return self._parse_bytestream(fp, options) 
Example 55
Project: NiujiaoDebugger   Author: MrSrc   File: saxutils.py    GNU General Public License v3.0 5 votes vote down vote up
def prepare_input_source(source, base=""):
    """This function takes an InputSource and an optional base URL and
    returns a fully resolved InputSource object ready for reading."""

    if isinstance(source, str):
        source = xmlreader.InputSource(source)
    elif hasattr(source, "read"):
        f = source
        source = xmlreader.InputSource()
        if isinstance(f.read(0), str):
            source.setCharacterStream(f)
        else:
            source.setByteStream(f)
        if hasattr(f, "name") and isinstance(f.name, str):
            source.setSystemId(f.name)

    if source.getCharacterStream() is None and source.getByteStream() is None:
        sysid = source.getSystemId()
        basehead = os.path.dirname(os.path.normpath(base))
        sysidfilename = os.path.join(basehead, sysid)
        if os.path.isfile(sysidfilename):
            source.setSystemId(sysidfilename)
            f = open(sysidfilename, "rb")
        else:
            source.setSystemId(urllib.parse.urljoin(base, sysid))
            f = urllib.request.urlopen(source.getSystemId())

        source.setByteStream(f)

    return source 
Example 56
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_qsl(self):
        for orig, expect in parse_qsl_test_cases:
            result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
            self.assertEqual(result, expect, "Error parsing %r" % orig)
            expect_without_blanks = [v for v in expect if len(v[1])]
            result = urllib.parse.parse_qsl(orig, keep_blank_values=False)
            self.assertEqual(result, expect_without_blanks,
                            "Error parsing %r" % orig) 
Example 57
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_qs(self):
        for orig, expect in parse_qs_test_cases:
            result = urllib.parse.parse_qs(orig, keep_blank_values=True)
            self.assertEqual(result, expect, "Error parsing %r" % orig)
            expect_without_blanks = {v: expect[v]
                                     for v in expect if len(expect[v][0])}
            result = urllib.parse.parse_qs(orig, keep_blank_values=False)
            self.assertEqual(result, expect_without_blanks,
                            "Error parsing %r" % orig) 
Example 58
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_http_roundtrips(self):
        # urllib.parse.urlsplit treats 'http:' as an optimized special case,
        # so we test both 'http:' and 'https:' in all the following.
        # Three cheers for white box knowledge!
        str_cases = [
            ('://www.python.org',
             ('www.python.org', '', '', '', ''),
             ('www.python.org', '', '', '')),
            ('://www.python.org#abc',
             ('www.python.org', '', '', '', 'abc'),
             ('www.python.org', '', '', 'abc')),
            ('://www.python.org?q=abc',
             ('www.python.org', '', '', 'q=abc', ''),
             ('www.python.org', '', 'q=abc', '')),
            ('://www.python.org/#abc',
             ('www.python.org', '/', '', '', 'abc'),
             ('www.python.org', '/', '', 'abc')),
            ('://a/b/c/d;p?q#f',
             ('a', '/b/c/d', 'p', 'q', 'f'),
             ('a', '/b/c/d;p', 'q', 'f')),
            ]
        def _encode(t):
            return (t[0].encode('ascii'),
                    tuple(x.encode('ascii') for x in t[1]),
                    tuple(x.encode('ascii') for x in t[2]))
        bytes_cases = [_encode(x) for x in str_cases]
        str_schemes = ('http', 'https')
        bytes_schemes = (b'http', b'https')
        str_tests = str_schemes, str_cases
        bytes_tests = bytes_schemes, bytes_cases
        for schemes, test_cases in (str_tests, bytes_tests):
            for scheme in schemes:
                for url, parsed, split in test_cases:
                    url = scheme + url
                    parsed = (scheme,) + parsed
                    split = (scheme,) + split
                    self.checkRoundtrips(url, parsed, split) 
Example 59
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_unparse_parse(self):
        str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]
        bytes_cases = [x.encode('ascii') for x in str_cases]
        for u in str_cases + bytes_cases:
            self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u)
            self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) 
Example 60
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_RFC2368(self):
        # Issue 11467: path that starts with a number is not parsed correctly
        self.assertEqual(urllib.parse.urlparse('mailto:1337@example.org'),
                ('mailto', '', '1337@example.org', '', '', '')) 
Example 61
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_urlsplit_scoped_IPv6(self):
        p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234')
        self.assertEqual(p.hostname, "fe80::822a:a8ff:fe49:470c%tESt")
        self.assertEqual(p.netloc, '[FE80::822a:a8ff:fe49:470c%tESt]:1234')

        p = urllib.parse.urlsplit(b'http://[FE80::822a:a8ff:fe49:470c%tESt]:1234')
        self.assertEqual(p.hostname, b"fe80::822a:a8ff:fe49:470c%tESt")
        self.assertEqual(p.netloc, b'[FE80::822a:a8ff:fe49:470c%tESt]:1234') 
Example 62
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_attributes_without_netloc(self):
        # This example is straight from RFC 3261.  It looks like it
        # should allow the username, hostname, and port to be filled
        # in, but doesn't.  Since it's a URI and doesn't use the
        # scheme://netloc syntax, the netloc and related attributes
        # should be left empty.
        uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
        p = urllib.parse.urlsplit(uri)
        self.assertEqual(p.netloc, "")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri)

        p = urllib.parse.urlparse(uri)
        self.assertEqual(p.netloc, "")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri)

        # You guessed it, repeating the test with bytes input
        uri = b"sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
        p = urllib.parse.urlsplit(uri)
        self.assertEqual(p.netloc, b"")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri)

        p = urllib.parse.urlparse(uri)
        self.assertEqual(p.netloc, b"")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri) 
Example 63
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_noslash(self):
        # Issue 1637: http://foo.com?query is legal
        self.assertEqual(urllib.parse.urlparse("http://example.com?blahblah=/foo"),
                         ('http', 'example.com', '', '', 'blahblah=/foo', ''))
        self.assertEqual(urllib.parse.urlparse(b"http://example.com?blahblah=/foo"),
                         (b'http', b'example.com', b'', b'', b'blahblah=/foo', b'')) 
Example 64
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_portseparator(self):
        # Issue 754016 makes changes for port separator ':' from scheme separator
        self.assertEqual(urllib.parse.urlparse("path:80"),
                ('','','path:80','','',''))
        self.assertEqual(urllib.parse.urlparse("http:"),('http','','','','',''))
        self.assertEqual(urllib.parse.urlparse("https:"),('https','','','','',''))
        self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
                ('http','www.python.org:80','','','',''))
        # As usual, need to check bytes input as well
        self.assertEqual(urllib.parse.urlparse(b"path:80"),
                (b'',b'',b'path:80',b'',b'',b''))
        self.assertEqual(urllib.parse.urlparse(b"http:"),(b'http',b'',b'',b'',b'',b''))
        self.assertEqual(urllib.parse.urlparse(b"https:"),(b'https',b'',b'',b'',b'',b''))
        self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
                (b'http',b'www.python.org:80',b'',b'',b'',b'')) 
Example 65
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_usingsys(self):
        # Issue 3314: sys module is used in the error
        self.assertRaises(TypeError, urllib.parse.urlencode, "foo") 
Example 66
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_default_scheme(self):
        # Exercise the scheme parameter of urlparse() and urlsplit()
        for func in (urllib.parse.urlparse, urllib.parse.urlsplit):
            with self.subTest(function=func):
                result = func("http://example.net/", "ftp")
                self.assertEqual(result.scheme, "http")
                result = func(b"http://example.net/", b"ftp")
                self.assertEqual(result.scheme, b"http")
                self.assertEqual(func("path", "ftp").scheme, "ftp")
                self.assertEqual(func("path", scheme="ftp").scheme, "ftp")
                self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp")
                self.assertEqual(func("path").scheme, "")
                self.assertEqual(func(b"path").scheme, b"")
                self.assertEqual(func(b"path", "").scheme, b"") 
Example 67
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_parse_fragments(self):
        # Exercise the allow_fragments parameter of urlparse() and urlsplit()
        tests = (
            ("http:#frag", "path", "frag"),
            ("//example.net#frag", "path", "frag"),
            ("index.html#frag", "path", "frag"),
            (";a=b#frag", "params", "frag"),
            ("?a=b#frag", "query", "frag"),
            ("#frag", "path", "frag"),
            ("abc#@frag", "path", "@frag"),
            ("//abc#@frag", "path", "@frag"),
            ("//abc:80#@frag", "path", "@frag"),
            ("//abc#@frag:80", "path", "@frag:80"),
        )
        for url, attr, expected_frag in tests:
            for func in (urllib.parse.urlparse, urllib.parse.urlsplit):
                if attr == "params" and func is urllib.parse.urlsplit:
                    attr = "path"
                with self.subTest(url=url, function=func):
                    result = func(url, allow_fragments=False)
                    self.assertEqual(result.fragment, "")
                    self.assertTrue(
                            getattr(result, attr).endswith("#" + expected_frag))
                    self.assertEqual(func(url, "", False).fragment, "")

                    result = func(url, allow_fragments=True)
                    self.assertEqual(result.fragment, expected_frag)
                    self.assertFalse(
                            getattr(result, attr).endswith(expected_frag))
                    self.assertEqual(func(url, "", True).fragment,
                                     expected_frag)
                    self.assertEqual(func(url).fragment, expected_frag) 
Example 68
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 5 votes vote down vote up
def test_result_pairs(self):
        # Check encoding and decoding between result pairs
        result_types = [
          urllib.parse.DefragResult,
          urllib.parse.SplitResult,
          urllib.parse.ParseResult,
        ]
        for result_type in result_types:
            self._check_result_type(result_type) 
Example 69
Project: fs_image   Author: facebookincubator   File: repo_downloader.py    MIT License 4 votes vote down vote up
def _download_repodata(
        self, repodata: 'Repodata', *, is_primary: bool
    ) -> Tuple[bool, str, Optional[List[Rpm]]]:
        '''
          - Returns True only if we just downloaded & stored this Repodata.
          - Returns our new storage_id, or the previous one from the DB.
          - For the selected primary repodata, returns a list of RPMs.
            Returns None for all others.
        '''
        # We only need to download the repodata if is not already in the DB,
        # or if it is primary (so we can parse it for RPMs).
        with self._repo_db_ctx as repo_db:
            storage_id = repo_db.get_storage_id(self._repodata_table, repodata)
        if is_primary:
            rpms = []
        elif storage_id:
            return False, storage_id, None  # Nothing stored, not primary
        else:
            rpms = None

        with (
            # We will parse the selected primary file to discover the RPMs.
            get_rpm_parser(repodata) if is_primary else nullcontext()
        ) as rpm_parser, (
            # Read the primary from storage if we already have an ID --
            # downloading is more likely to fail due to repo updates.
            self._storage.reader(storage_id) if storage_id
                else self._download(repodata.location)
        ) as input, (
            # Only write to storage if it's not already there.
            self._storage.writer() if not storage_id else nullcontext()
        ) as output:
            log.info(f'Fetching {repodata}')
            for chunk in _verify_chunk_stream(
                read_chunks(input, BUFFER_BYTES),
                [repodata.checksum],
                repodata.size,
                repodata.location,
            ):  # May raise a ReportableError
                if output:
                    output.write(chunk)
                if rpm_parser:
                    try:
                        rpms.extend(rpm_parser.feed(chunk))
                    except Exception as ex:
                        raise RepodataParseError((repodata.location, ex))
            # Must commit from inside the output context to get a storage_id.
            if output:
                return True, output.commit(), rpms
        # The repodata was already stored, and we parsed it for RPMs.
        assert storage_id is not None
        return False, storage_id, rpms 
Example 70
Project: PheKnowLator   Author: callahantiff   File: EdgeDictionary.py    Apache License 2.0 4 votes vote down vote up
def queries_ontologies(data_file):
        """Takes a string representing a path/file name to an ontology. The function uses the RDFLib library
        and creates a graph. The graph is then queried to return all classes and their database cross-references. The
        function returns a list of query results

        Args:
            data_file (str): A filepath pointing to an ontology saved in an '.owl' file.

         Returns:
            A dictionary of results returned from mapping identifiers.

        Raises:
            An exception is raised if the generated dictionary does not have the same number of entries as the
            ontology graph.
         """

        # read in ontology as graph
        graph = Graph()
        graph.parse(data_file)

        # query graph to get all cross-referenced sources
        results = graph.query(
            """SELECT DISTINCT ?source ?c
               WHERE {
                  ?c rdf:type owl:Class .
                  ?c oboInOwl:hasDbXref ?source .}
               """, initNs={"rdf": 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
                            "owl": 'http://www.w3.org/2002/07/owl#',
                            "oboInOwl": 'http://www.geneontology.org/formats/oboInOwl#'})

        # convert results to dictionary
        ont_results = {}
        for res in list(results):
            if str(res[0]).split(':')[-1] in ont_results.keys():
                ont_results[str(res[0]).split(':')[-1]].append(str(res[1]))

            else:
                ont_results[str(res[0]).split(':')[-1]] = [str(res[1])]

        # CHECK - all URLs returned an data file
        if len(list(ont_results)) == 0:
            raise Exception('ERROR: Query returned no data')
        else:
            return ont_results 
Example 71
Project: colorcore   Author: martexcoin   File: routing.py    MIT License 4 votes vote down vote up
def handle_request(self, message, payload):
        try:
            url = re.search('^/(?P<operation>\w+)$', message.path)
            if url is None:
                return (yield from self.error(102, 'The request path is invalid', message))

            # Get the operation function corresponding to the URL path
            operation_name = url.group('operation')
            operation = getattr(self.controller, operation_name, None)

            if operation_name == '' or operation_name[0] == '_' or operation is None:
                return (yield from self.error(
                    103, 'The operation name {name} is invalid'.format(name=operation_name), message))

            # Read the POST body
            post_data = yield from payload.read()
            post_vars = {str(k, 'utf-8'): str(v[0], 'utf-8') for k, v in urllib.parse.parse_qs(post_data).items()}

            tx_parser = Router.get_transaction_formatter(post_vars.pop('txformat', 'json'))

            controller = self.controller(self.configuration, self.cache_factory, tx_parser, self.event_loop)

            try:
                result = yield from operation(controller, **post_vars)
            except TypeError:
                return (yield from self.error(104, 'Invalid parameters provided', message))
            except ControllerError as error:
                return (yield from self.error(201, str(error), message))
            except openassets.transactions.TransactionBuilderError as error:
                return (yield from self.error(301, type(error).__name__, message))
            except NotImplementedError as error:
                return (yield from self.error(202, str(error), message))

            response = self.create_response(200, message)
            yield from self.json_response(response, result)

            if response.keep_alive():
                self.keep_alive(True)

        except Exception as exception:
            response = self.create_response(500, message)
            yield from self.json_response(
                response, {'error': {'code': 0, 'message': 'Internal server error', 'details': str(exception)}}) 
Example 72
Project: steam_workshop_downloader   Author: Geam   File: workshop.py    MIT License 4 votes vote down vote up
def get_plugins_id_from_collections_list (collections_id_list):
    """Ask the steam api for every plugin in the collection(s) and
    subcollection(s)
    Will return:
        - error:
            - None if no error encounter
            - the error an error occur
        - an array of all plugins id
        - an array of the valid collection(s) id given as arg (and not all the
            subcollection)
    Return error(error), array(plugin_id), array(valid_collections_id)
    """
    valid_collections = []
    sub_collection = []
    plugins_id_list = []
    error = None
    data = const_data['collection']
    data['collectioncount'] = len(collections_id_list)
    for idx, collection_id in enumerate(collections_id_list):
        data['publishedfileids[' + str(idx) + ']'] = collection_id
    encode_data = urllib.parse.urlencode(data).encode('ascii')
    try:
        response = urllib.request.urlopen(const_urls['collection'], encode_data)
    except HTTPError as e:
        print("Server return " + str(e.code) + " error")
        error = e
    except URLError as e:
        print("Can't reach server: " + e.reason)
        error = e
    else:
        json_response = json.loads(response.read().decode('utf-8'))
        for collection in json_response['response']['collectiondetails']:
            if 'children' in collection:
                # if collection is a valid one
                valid_collections.append(collection['publishedfileid'])
                for item in collection['children']:
                    if item['filetype'] == 0:   # children is a plugin
                        plugins_id_list.append(item['publishedfileid'])
                    elif item['filetype'] == 2: # childre is a collection
                        sub_collection.append(item['publishedfileid'])
                    else:                       # unknown type
                        print("Unrecognised filetype: " + str(item['filetype']))
        if len(sub_collection) > 0:
            error, plugins_id_list_temp, o = \
                get_plugins_id_from_collections_list(sub_collection)
            if error == None:
                plugins_id_list += plugins_id_list_temp
    return error, plugins_id_list, valid_collections 
Example 73
Project: YaYaGen   Author: jimmy-sonny   File: url_checker.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def detected_VT(self, domain, ipaddr=False, nodb=False):
        """
        Return True if the domain is known and detected by VT
        (that is, it's potentially malicious)
        """
        try:
            if not re.match('.+\..+', domain):
                return False

            if not UrlChecker.__vtapikey:
                return False
                
            detections = self.__fetch(domain)
            # Not in DB
            if (detections is None) or nodb:
                log.debug("querying VT: %s", domain)
                if ipaddr:
                    parameters = {'ip': domain,
                                  'apikey': UrlChecker.__vtapikey}
                    vt_url = VT_URL_IPADDR
                else:
                    parameters = {'domain': domain,
                                  'apikey': UrlChecker.__vtapikey}
                    vt_url = VT_URL_TEXT
                param = urllib.parse.urlencode(parameters)
                vurl = '%s?%s' % (vt_url, param)
                response = urllib.request.urlopen(vurl).read()
                if len(response) == 0:
                    log.error("VT error - (probably API limit exceeded)")
                    return False
                jresponse = json.loads(response)
                if jresponse['response_code'] == 0:
                    # Item not present in VirusTotal's dataset
                    self.__store(domain, [])
                    return False
                detections = pyjq.all(
                    '.detected_urls[] | .positives', jresponse)
                self.__store(domain, detections)

            if len(detections) >= UrlChecker.__min_detections:
                # Domain is detected
                return True
            return False

        except Exception:
            log.error("Error while querying domain: %s", domain)
            return False 
Example 74
Project: WebMap   Author: SECUREFOREST   File: api.py    GNU General Public License v3.0 4 votes vote down vote up
def getCVE(request):
	res = {}

	if 'auth' not in request.session:
		return False

	if request.method == "POST":
		scanfilemd5 = hashlib.md5(str(request.session['scanfile']).encode('utf-8')).hexdigest()
		cveproc = os.popen('python3 /opt/nmapdashboard/nmapreport/nmap/cve.py '+request.session['scanfile'])
		res['cveout'] = cveproc.read()
		cveproc.close()

		return HttpResponse(json.dumps(res), content_type="application/json")

		#hostmd5 = hashlib.md5(str(request.POST['host']).encode('utf-8')).hexdigest()
		#portmd5 = hashlib.md5(str(request.POST['port']).encode('utf-8')).hexdigest()

		# request.POST['host']

		cpe = json.loads(base64.b64decode(urllib.parse.unquote(request.POST['cpe'])).decode('ascii'))

		for cpestr in cpe:
			r = requests.get('http://cve.circl.lu/api/cvefor/'+cpestr)
			cvejson = r.json()

			for host in cpe[cpestr]:
				hostmd5 = hashlib.md5(str(host).encode('utf-8')).hexdigest()
				if type(cvejson) is list and len(cvejson) > 0:
					res[host] = cvejson[0]
					f = open('/opt/notes/'+scanfilemd5+'_'+hostmd5+'.cve', 'w')
					f.write(json.dumps(cvejson))
					f.close()

		return HttpResponse(json.dumps(res), content_type="application/json")

		r = requests.get('http://cve.circl.lu/api/cvefor/'+request.POST['cpe'])

		if request.POST['host'] not in res:
			res[request.POST['host']] = {}

		cvejson = r.json()

		if type(cvejson) is list and len(cvejson) > 0:
			res[request.POST['host']][request.POST['port']] = cvejson[0]
			f = open('/opt/notes/'+scanfilemd5+'_'+hostmd5+'.cve', 'w')
			f.write(json.dumps(cvejson))
			f.close()

		return HttpResponse(json.dumps(res), content_type="application/json") 
Example 75
Project: NiujiaoDebugger   Author: MrSrc   File: utils.py    GNU General Public License v3.0 4 votes vote down vote up
def decode_params(params):
    """Decode parameters list according to RFC 2231.

    params is a sequence of 2-tuples containing (param name, string value).
    """
    # Copy params so we don't mess with the original
    params = params[:]
    new_params = []
    # Map parameter's name to a list of continuations.  The values are a
    # 3-tuple of the continuation number, the string value, and a flag
    # specifying whether a particular segment is %-encoded.
    rfc2231_params = {}
    name, value = params.pop(0)
    new_params.append((name, value))
    while params:
        name, value = params.pop(0)
        if name.endswith('*'):
            encoded = True
        else:
            encoded = False
        value = unquote(value)
        mo = rfc2231_continuation.match(name)
        if mo:
            name, num = mo.group('name', 'num')
            if num is not None:
                num = int(num)
            rfc2231_params.setdefault(name, []).append((num, value, encoded))
        else:
            new_params.append((name, '"%s"' % quote(value)))
    if rfc2231_params:
        for name, continuations in rfc2231_params.items():
            value = []
            extended = False
            # Sort by number
            continuations.sort()
            # And now append all values in numerical order, converting
            # %-encodings for the encoded segments.  If any of the
            # continuation names ends in a *, then the entire string, after
            # decoding segments and concatenating, must have the charset and
            # language specifiers at the beginning of the string.
            for num, s, encoded in continuations:
                if encoded:
                    # Decode as "latin-1", so the characters in s directly
                    # represent the percent-encoded octet values.
                    # collapse_rfc2231_value treats this as an octet sequence.
                    s = urllib.parse.unquote(s, encoding="latin-1")
                    extended = True
                value.append(s)
            value = quote(EMPTYSTRING.join(value))
            if extended:
                charset, language, value = decode_rfc2231(value)
                new_params.append((name, (charset, language, '"%s"' % value)))
            else:
                new_params.append((name, '"%s"' % value))
    return new_params 
Example 76
Project: NiujiaoDebugger   Author: MrSrc   File: mimetypes.py    GNU General Public License v3.0 4 votes vote down vote up
def guess_type(self, url, strict=True):
        """Guess the type of a file based on its URL.

        Return value is a tuple (type, encoding) where type is None if
        the type can't be guessed (no or unknown suffix) or a string
        of the form type/subtype, usable for a MIME Content-type
        header; and encoding is None for no encoding or the name of
        the program used to encode (e.g. compress or gzip).  The
        mappings are table driven.  Encoding suffixes are case
        sensitive; type suffixes are first tried case sensitive, then
        case insensitive.

        The suffixes .tgz, .taz and .tz (case sensitive!) are all
        mapped to '.tar.gz'.  (This is table-driven too, using the
        dictionary suffix_map.)

        Optional `strict' argument when False adds a bunch of commonly found,
        but non-standard types.
        """
        scheme, url = urllib.parse.splittype(url)
        if scheme == 'data':
            # syntax of data URLs:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            # type/subtype defaults to "text/plain"
            comma = url.find(',')
            if comma < 0:
                # bad data URL
                return None, None
            semi = url.find(';', 0, comma)
            if semi >= 0:
                type = url[:semi]
            else:
                type = url[:comma]
            if '=' in type or '/' not in type:
                type = 'text/plain'
            return type, None           # never compressed, so encoding is None
        base, ext = posixpath.splitext(url)
        while ext in self.suffix_map:
            base, ext = posixpath.splitext(base + self.suffix_map[ext])
        if ext in self.encodings_map:
            encoding = self.encodings_map[ext]
            base, ext = posixpath.splitext(base)
        else:
            encoding = None
        types_map = self.types_map[True]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        elif strict:
            return None, encoding
        types_map = self.types_map[False]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        else:
            return None, encoding 
Example 77
Project: NiujiaoDebugger   Author: MrSrc   File: server.py    GNU General Public License v3.0 4 votes vote down vote up
def list_directory(self, path):
        """Helper to produce a directory listing (absent index.html).

        Return value is either a file object, or None (indicating an
        error).  In either case, the headers are sent, making the
        interface the same as for send_head().

        """
        try:
            list = os.listdir(path)
        except OSError:
            self.send_error(
                HTTPStatus.NOT_FOUND,
                "No permission to list directory")
            return None
        list.sort(key=lambda a: a.lower())
        r = []
        try:
            displaypath = urllib.parse.unquote(self.path,
                                               errors='surrogatepass')
        except UnicodeDecodeError:
            displaypath = urllib.parse.unquote(path)
        displaypath = html.escape(displaypath, quote=False)
        enc = sys.getfilesystemencoding()
        title = 'Directory listing for %s' % displaypath
        r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
                 '"http://www.w3.org/TR/html4/strict.dtd">')
        r.append('<html>\n<head>')
        r.append('<meta http-equiv="Content-Type" '
                 'content="text/html; charset=%s">' % enc)
        r.append('<title>%s</title>\n</head>' % title)
        r.append('<body>\n<h1>%s</h1>' % title)
        r.append('<hr>\n<ul>')
        for name in list:
            fullname = os.path.join(path, name)
            displayname = linkname = name
            # Append / for directories or @ for symbolic links
            if os.path.isdir(fullname):
                displayname = name + "/"
                linkname = name + "/"
            if os.path.islink(fullname):
                displayname = name + "@"
                # Note: a link to a directory displays with @ and links with /
            r.append('<li><a href="%s">%s</a></li>'
                    % (urllib.parse.quote(linkname,
                                          errors='surrogatepass'),
                       html.escape(displayname, quote=False)))
        r.append('</ul>\n<hr>\n</body>\n</html>\n')
        encoded = '\n'.join(r).encode(enc, 'surrogateescape')
        f = io.BytesIO()
        f.write(encoded)
        f.seek(0)
        self.send_response(HTTPStatus.OK)
        self.send_header("Content-type", "text/html; charset=%s" % enc)
        self.send_header("Content-Length", str(len(encoded)))
        self.end_headers()
        return f 
Example 78
Project: NiujiaoDebugger   Author: MrSrc   File: server.py    GNU General Public License v3.0 4 votes vote down vote up
def _url_collapse_path(path):
    """
    Given a URL path, remove extra '/'s and '.' path elements and collapse
    any '..' references and returns a collapsed path.

    Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
    The utility of this function is limited to is_cgi method and helps
    preventing some security attacks.

    Returns: The reconstituted URL, which will always start with a '/'.

    Raises: IndexError if too many '..' occur within the path.

    """
    # Query component should not be involved.
    path, _, query = path.partition('?')
    path = urllib.parse.unquote(path)

    # Similar to os.path.split(os.path.normpath(path)) but specific to URL
    # path semantics rather than local operating system semantics.
    path_parts = path.split('/')
    head_parts = []
    for part in path_parts[:-1]:
        if part == '..':
            head_parts.pop() # IndexError if more '..' than prior parts
        elif part and part != '.':
            head_parts.append( part )
    if path_parts:
        tail_part = path_parts.pop()
        if tail_part:
            if tail_part == '..':
                head_parts.pop()
                tail_part = ''
            elif tail_part == '.':
                tail_part = ''
    else:
        tail_part = ''

    if query:
        tail_part = '?'.join((tail_part, query))

    splitpath = ('/' + '/'.join(head_parts), tail_part)
    collapsed_path = "/".join(splitpath)

    return collapsed_path 
Example 79
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 4 votes vote down vote up
def checkRoundtrips(self, url, parsed, split):
        result = urllib.parse.urlparse(url)
        self.assertEqual(result, parsed)
        t = (result.scheme, result.netloc, result.path,
             result.params, result.query, result.fragment)
        self.assertEqual(t, parsed)
        # put it back together and it should be the same
        result2 = urllib.parse.urlunparse(result)
        self.assertEqual(result2, url)
        self.assertEqual(result2, result.geturl())

        # the result of geturl() is a fixpoint; we can always parse it
        # again to get the same result:
        result3 = urllib.parse.urlparse(result.geturl())
        self.assertEqual(result3.geturl(), result.geturl())
        self.assertEqual(result3,          result)
        self.assertEqual(result3.scheme,   result.scheme)
        self.assertEqual(result3.netloc,   result.netloc)
        self.assertEqual(result3.path,     result.path)
        self.assertEqual(result3.params,   result.params)
        self.assertEqual(result3.query,    result.query)
        self.assertEqual(result3.fragment, result.fragment)
        self.assertEqual(result3.username, result.username)
        self.assertEqual(result3.password, result.password)
        self.assertEqual(result3.hostname, result.hostname)
        self.assertEqual(result3.port,     result.port)

        # check the roundtrip using urlsplit() as well
        result = urllib.parse.urlsplit(url)
        self.assertEqual(result, split)
        t = (result.scheme, result.netloc, result.path,
             result.query, result.fragment)
        self.assertEqual(t, split)
        result2 = urllib.parse.urlunsplit(result)
        self.assertEqual(result2, url)
        self.assertEqual(result2, result.geturl())

        # check the fixpoint property of re-parsing the result of geturl()
        result3 = urllib.parse.urlsplit(result.geturl())
        self.assertEqual(result3.geturl(), result.geturl())
        self.assertEqual(result3,          result)
        self.assertEqual(result3.scheme,   result.scheme)
        self.assertEqual(result3.netloc,   result.netloc)
        self.assertEqual(result3.path,     result.path)
        self.assertEqual(result3.query,    result.query)
        self.assertEqual(result3.fragment, result.fragment)
        self.assertEqual(result3.username, result.username)
        self.assertEqual(result3.password, result.password)
        self.assertEqual(result3.hostname, result.hostname)
        self.assertEqual(result3.port,     result.port) 
Example 80
Project: NiujiaoDebugger   Author: MrSrc   File: test_urlparse.py    GNU General Public License v3.0 4 votes vote down vote up
def test_RFC2732(self):
        str_cases = [
            ('http://Test.python.org:5432/foo/', 'test.python.org', 5432),
            ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432),
            ('http://[::1]:5432/foo/', '::1', 5432),
            ('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432),
            ('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432),
            ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/',
             'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432),
            ('http://[::12.34.56.78]:5432/foo/', '::12.34.56.78', 5432),
            ('http://[::ffff:12.34.56.78]:5432/foo/',
             '::ffff:12.34.56.78', 5432),
            ('http://Test.python.org/foo/', 'test.python.org', None),
            ('http://12.34.56.78/foo/', '12.34.56.78', None),
            ('http://[::1]/foo/', '::1', None),
            ('http://[dead:beef::1]/foo/', 'dead:beef::1', None),
            ('http://[dead:beef::]/foo/', 'dead:beef::', None),
            ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/',
             'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
            ('http://[::12.34.56.78]/foo/', '::12.34.56.78', None),
            ('http://[::ffff:12.34.56.78]/foo/',
             '::ffff:12.34.56.78', None),
            ('http://Test.python.org:/foo/', 'test.python.org', None),
            ('http://12.34.56.78:/foo/', '12.34.56.78', None),
            ('http://[::1]:/foo/', '::1', None),
            ('http://[dead:beef::1]:/foo/', 'dead:beef::1', None),
            ('http://[dead:beef::]:/foo/', 'dead:beef::', None),
            ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:/foo/',
             'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
            ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None),
            ('http://[::ffff:12.34.56.78]:/foo/',
             '::ffff:12.34.56.78', None),
            ]
        def _encode(t):
            return t[0].encode('ascii'), t[1].encode('ascii'), t[2]
        bytes_cases = [_encode(x) for x in str_cases]
        for url, hostname, port in str_cases + bytes_cases:
            urlparsed = urllib.parse.urlparse(url)
            self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port))

        str_cases = [
                'http://::12.34.56.78]/',
                'http://[::1/foo/',
                'ftp://[::1/foo/bad]/bad',
                'http://[::1/foo/bad]/bad',
                'http://[::ffff:12.34.56.78']
        bytes_cases = [x.encode('ascii') for x in str_cases]
        for invalid_url in str_cases + bytes_cases:
            self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url)