Python urlparse.urlunparse() Examples

The following are code examples for showing how to use urlparse.urlunparse(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: webchecker.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def getlinkinfos(self):
        # File reading is done in __init__() routine.  Store parser in
        # local variable to indicate success of parsing.

        # If no parser was stored, fail.
        if not self.parser: return []

        rawlinks = self.parser.getlinks()
        base = urlparse.urljoin(self.url, self.parser.getbase() or "")
        infos = []
        for rawlink in rawlinks:
            t = urlparse.urlparse(rawlink)
            # DON'T DISCARD THE FRAGMENT! Instead, include
            # it in the tuples which are returned. See Checker.dopage().
            fragment = t[-1]
            t = t[:-1] + ('',)
            rawlink = urlparse.urlunparse(t)
            link = urlparse.urljoin(base, rawlink)
            infos.append((link, rawlink, fragment))

        return infos 
Example 2
Project: flasky   Author: RoseOu   File: flask_login.py    MIT License 6 votes vote down vote up
def make_next_param(login_url, current_url):
    '''
    Reduces the scheme and host from a given URL so it can be passed to
    the given `login` URL more efficiently.

    :param login_url: The login URL being redirected to.
    :type login_url: str
    :param current_url: The URL to reduce.
    :type current_url: str
    '''
    l = urlparse(login_url)
    c = urlparse(current_url)

    if (not l.scheme or l.scheme == c.scheme) and \
            (not l.netloc or l.netloc == c.netloc):
        return urlunparse(('', '', c.path, c.params, c.query, ''))
    return current_url 
Example 3
Project: presto-admin   Author: prestosql   File: workers.py    Apache License 2.0 6 votes vote down vote up
def validate(conf):
        validate_presto_conf(conf)
        if 'coordinator' not in conf['config.properties']:
            raise ConfigurationError('Must specify coordinator=false in '
                                     'worker\'s config.properties')
        if conf['config.properties']['coordinator'] != 'false':
            raise ConfigurationError('Coordinator must be false in the '
                                     'worker\'s config.properties')
        uri = urlparse.urlparse(conf['config.properties']['discovery.uri'])
        if Worker.is_localhost(uri.hostname) and len(env.roledefs['all']) > 1:
            raise ConfigurationError(
                'discovery.uri should not be localhost in a '
                'multi-node cluster, but found ' + urlparse.urlunparse(uri) +
                '.  You may have encountered this error by '
                'choosing a coordinator that is localhost and a worker that '
                'is not.  The default discovery-uri is '
                'http://<coordinator>:8080')
        return conf 
Example 4
Project: openhatch   Author: campbe13   File: url.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
        encoding=None):
    """Canonicalize the given url by applying the following procedures:

    - sort query arguments, first by key, then by value
    - percent encode paths and query arguments. non-ASCII characters are
      percent-encoded using UTF-8 (RFC-3986)
    - normalize all spaces (in query arguments) '+' (plus symbol)
    - normalize percent encodings case (%2f -> %2F)
    - remove query arguments with blank values (unless keep_blank_values is True)
    - remove fragments (unless keep_fragments is True)

    The url passed can be a str or unicode, while the url returned is always a
    str.

    For examples see the tests in scrapy.tests.test_utils_url
    """

    scheme, netloc, path, params, query, fragment = parse_url(url)
    keyvals = cgi.parse_qsl(query, keep_blank_values)
    keyvals.sort()
    query = urllib.urlencode(keyvals)
    path = safe_url_string(urllib.unquote(path))
    fragment = '' if not keep_fragments else fragment
    return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment)) 
Example 5
Project: recipes-py   Author: luci   File: streamname.py    Apache License 2.0 6 votes vote down vote up
def get_logdog_viewer_url(host, project, *stream_paths):
  """Returns (str): The LogDog viewer URL for the named stream(s).

  Args:
    host (str): The name of the Coordiantor host.
    project (str): The project name.
    stream_paths: A set of StreamPath instances for the stream paths to
        generate the URL for.
  """
  return urlparse.urlunparse((
    'https', # Scheme
    host, # netloc
    'v/', # path
    '', # params
    '&'.join(('s=%s' % (urllib.quote('%s/%s' % (project, path), safe=''))
              for path in stream_paths)), # query
    '', # fragment
  )) 
Example 6
Project: analyzer-website   Author: santiagolizardo   File: __init__.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def url(self, value):
        self.__dict__['url'] = value
        if value is not None:
            scheme, netloc, path, params, query, fragment = urlparse.urlparse(value)

            # Exclude default port numbers.
            if scheme == 'http' and netloc[-3:] == ':80':
                netloc = netloc[:-3]
            elif scheme == 'https' and netloc[-4:] == ':443':
                netloc = netloc[:-4]
            if scheme not in ('http', 'https'):
                raise ValueError("Unsupported URL %s (%s)." % (value, scheme))

            # Normalized URL excludes params, query, and fragment.
            self.normalized_url = urlparse.urlunparse((scheme, netloc, path, None, None, None))
        else:
            self.normalized_url = None
            self.__dict__['url'] = None 
Example 7
Project: Flask_Blog   Author: sugarguo   File: flask_login.py    GNU General Public License v3.0 6 votes vote down vote up
def make_next_param(login_url, current_url):
    '''
    Reduces the scheme and host from a given URL so it can be passed to
    the given `login` URL more efficiently.

    :param login_url: The login URL being redirected to.
    :type login_url: str
    :param current_url: The URL to reduce.
    :type current_url: str
    '''
    l = urlparse(login_url)
    c = urlparse(current_url)

    if (not l.scheme or l.scheme == c.scheme) and \
            (not l.netloc or l.netloc == c.netloc):
        return urlunparse(('', '', c.path, c.params, c.query, ''))
    return current_url 
Example 8
Project: Pansidong   Author: lightless233   File: WebSpider.py    GNU General Public License v3.0 6 votes vote down vote up
def format_url_param(url):
        url_st = urlparse.urlparse(url)
        queries = url_st.query
        if not queries:
            return
        new_queries = ""
        for eq in queries.split("&"):
            key = eq.split("=")[0]
            value = eq.split("=")[1]
            if value.isdigit():
                value = "<int>"
            new_queries += key + "=" + value + "&"
        new_queries = new_queries.strip("&")
        url = urlparse.urlunparse((
            url_st.scheme,
            url_st.netloc,
            url_st.path,
            url_st.params,
            new_queries,
            url_st.fragment,
        ))
        return url 
Example 9
Project: sphinx-javalink   Author: bluekeyes   File: ref.py    MIT License 6 votes vote down vote up
def _parse_docroot_str(srcdir, root):
    scheme, netloc, path = urlparse(root)[0:3]
    if not scheme:
        # assume local path; add trailing '/'s if missing
        root = os.path.join(root, '')
        absroot = os.path.join(abspath(srcdir, root), '')

        url = urljoin('file:///', pathname2url(absroot))
        base = pathname2url(root)
    else:
        path = path.rstrip('/') + '/'

        url = urlunparse((scheme, netloc, path, '', '', ''))
        base = url

    return (urljoin(url, 'package-list'), base) 
Example 10
Project: plugin.audio.subsonic   Author: basilfx   File: __init__.py    MIT License 6 votes vote down vote up
def _doBinReq(self, *args, **kwargs):
        """
        Intercept request URL to provide the URL of the item that is requested.

        If the URL is intercepted, the request is not executed. A username and
        password is added to provide direct access to the stream.
        """

        if self.intercept_url:
            parts = list(urlparse.urlparse(
                args[0].get_full_url() + "?" + args[0].data))
            parts[4] = dict(urlparse.parse_qsl(parts[4]))
            parts[4].update({"u": self.username, "p": self.password})
            parts[4] = urllib.urlencode(parts[4])

            return urlparse.urlunparse(parts)
        else:
            return super(SubsonicClient, self)._doBinReq(*args, **kwargs) 
Example 11
Project: ANALYSE-v1   Author: analyseuc3m   File: models.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _apply_cdn(self, image_urls):
        """
        Given a dict of resolutions -> urls, return a copy with CDN applied.

        If CDN does not exist or is disabled, just returns the original. The
        URLs that we store in CourseOverviewImageSet are all already top level
        paths, so we don't need to go through the /static remapping magic that
        happens with other course assets. We just need to add the CDN server if
        appropriate.
        """
        cdn_config = AssetBaseUrlConfig.current()
        if not cdn_config.enabled:
            return image_urls

        base_url = cdn_config.base_url

        return {
            resolution: urlunparse((None, base_url, url, None, None, None))
            for resolution, url in image_urls.items()
        } 
Example 12
Project: circlecli   Author: particledecay   File: circleapi.py    MIT License 6 votes vote down vote up
def _build_url(self, endpoint, params={}):
        """Return the full URL for the desired endpoint.

        Args:
            endpoint (str): the API endpoint after base URL
            params (dict): any params to include in the request

        Returns:
            (str) the full URL of the request
        """
        new_params = {'circle-token': self._token}
        new_params.update(params)

        parsed_url = urlparse(self._base_url)
        new_parse = ParseResult(scheme=parsed_url.scheme, netloc=parsed_url.netloc,
                                path='/'.join((parsed_url.path, endpoint)),
                                params='', query=urlencode(new_params),
                                fragment='')

        return urlunparse(new_parse) 
Example 13
Project: jbox   Author: jpush   File: flask_login.py    MIT License 6 votes vote down vote up
def make_next_param(login_url, current_url):
    '''
    Reduces the scheme and host from a given URL so it can be passed to
    the given `login` URL more efficiently.

    :param login_url: The login URL being redirected to.
    :type login_url: str
    :param current_url: The URL to reduce.
    :type current_url: str
    '''
    l = urlparse(login_url)
    c = urlparse(current_url)

    if (not l.scheme or l.scheme == c.scheme) and \
            (not l.netloc or l.netloc == c.netloc):
        return urlunparse(('', '', c.path, c.params, c.query, ''))
    return current_url 
Example 14
Project: rucio   Author: rucio   File: extract_agis_srm.py    Apache License 2.0 6 votes vote down vote up
def run():
    sites = requests.get('http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json').json()
    tmp = []  # needed because urlparse returns constant tuples
    res = []

    for site in sites:
        try:
            # let's hope the JSON schema doesn't change in the future
            tmp.append((site['token'], urlparse.urlparse(str(''.join([s for s in site['protocols'] if s.startswith('srm')][0] + [site['protocols'][s] for s in site['protocols'] if s.startswith('srm')][0][0][2])))))
        except:
            pass
    for t in tmp:
        u = list(t[1])
        v = urlparse.urlunparse(u)

        # Sites that do not have a prefix defined are removed
        if v.endswith('/'):
            res.append((v, str(t[0])))

    res.sort()
    for r in res:
        print '%s:%s' % (r[1], ':'.join(r[0].split(':')[1:])) 
Example 15
Project: omniduct   Author: airbnb   File: base.py    MIT License 6 votes vote down vote up
def get_local_uri(self, uri):
        """
        Convert a remote uri to a local one.

        This method takes a remote service uri accessible to the remote host and
        returns a local uri accessible directly on the local host, establishing
        any necessary port forwarding in the process.

        Args:
            uri (str): The remote uri to be made local.

        Returns:
            str: A local uri that tunnels all traffic to the remote host.
        """
        parsed_uri = urlparse(uri)
        return urlunparse(parsed_uri._replace(netloc='localhost:{}'.format(self.port_forward(parsed_uri.netloc)))) 
Example 16
Project: oa_qian   Author: sunqb   File: flask_login.py    Apache License 2.0 6 votes vote down vote up
def make_next_param(login_url, current_url):
    '''
    Reduces the scheme and host from a given URL so it can be passed to
    the given `login` URL more efficiently.

    :param login_url: The login URL being redirected to.
    :type login_url: str
    :param current_url: The URL to reduce.
    :type current_url: str
    '''
    l = urlparse(login_url)
    c = urlparse(current_url)

    if (not l.scheme or l.scheme == c.scheme) and \
            (not l.netloc or l.netloc == c.netloc):
        return urlunparse(('', '', c.path, c.params, c.query, ''))
    return current_url 
Example 17
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 6 votes vote down vote up
def render(self, context):
        urlp = list(urlparse(self.url.resolve(context)))
        qp = parse_qsl(urlp[4])
        for name, op, value in self.qs:
            name = smart_str(name)
            value = value.resolve(context)
            value = smart_str(value) if value is not None else None
            if op == '+=':
                qp = filter(lambda (n, v): not(n == name and v == value), qp)
                qp.append((name, value,))
            elif op == '-=':
                qp = filter(lambda (n, v): not(n == name and v == value), qp)
            elif op == '=':
                qp = filter(lambda (n, v): not(n == name), qp)
                if value is not None:
                    qp.append((name, value,))
 
        urlp[4] = urlencode(qp, True)
        url = urlunparse(urlp)
 
        if self.asvar:
            context[self.asvar] = url
            return ''
        else:
            return url 
Example 18
Project: pyblish-win   Author: pyblish   File: webchecker.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def addroot(self, root, add_to_do = 1):
        if root not in self.roots:
            troot = root
            scheme, netloc, path, params, query, fragment = \
                    urlparse.urlparse(root)
            i = path.rfind("/") + 1
            if 0 < i < len(path):
                path = path[:i]
                troot = urlparse.urlunparse((scheme, netloc, path,
                                             params, query, fragment))
            self.roots.append(troot)
            self.addrobot(root)
            if add_to_do:
                self.newlink((root, ""), ("<root>", root)) 
Example 19
Project: pyblish-win   Author: pyblish   File: robotparser.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def can_fetch(self, useragent, url):
        """using the parsed robots.txt decide if useragent can fetch url"""
        if self.disallow_all:
            return False
        if self.allow_all:
            return True

        # Until the robots.txt file has been read or found not
        # to exist, we must assume that no url is allowable.
        # This prevents false positives when a user erronenously
        # calls can_fetch() before calling read().
        if not self.last_checked:
            return False

        # search for given user agent matches
        # the first match counts
        parsed_url = urlparse.urlparse(urllib.unquote(url))
        url = urlparse.urlunparse(('', '', parsed_url.path,
            parsed_url.params, parsed_url.query, parsed_url.fragment))
        url = urllib.quote(url)
        if not url:
            url = "/"
        for entry in self.entries:
            if entry.applies_to(useragent):
                return entry.allowance(url)
        # try the default entry last
        if self.default_entry:
            return self.default_entry.allowance(url)
        # agent not found ==> access granted
        return True 
Example 20
Project: pyblish-win   Author: pyblish   File: robotparser.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, path, allowance):
        if path == '' and not allowance:
            # an empty value means allow all
            allowance = True
        path = urlparse.urlunparse(urlparse.urlparse(path))
        self.path = urllib.quote(path)
        self.allowance = allowance 
Example 21
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_unparse_parse(self):
        for u in ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]:
            self.assertEqual(urlparse.urlunsplit(urlparse.urlsplit(u)), u)
            self.assertEqual(urlparse.urlunparse(urlparse.urlparse(u)), u) 
Example 22
Project: pnp   Author: HazardDede   File: storage.py    MIT License 5 votes vote down vote up
def _make_raw_file_url(shared_url):
        try:
            import urlparse
            from urllib import urlencode
        except ImportError:  # For Python 3
            import urllib.parse as urlparse
            from urllib.parse import urlencode

        url_parts = list(urlparse.urlparse(shared_url))
        # Replace all query params with ?raw=1
        url_parts[4] = urlencode({'raw': '1'})
        return urlparse.urlunparse(url_parts) 
Example 23
Project: flasky   Author: RoseOu   File: pkg_resources.py    MIT License 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 24
Project: flasky   Author: RoseOu   File: flask_login.py    MIT License 5 votes vote down vote up
def login_url(login_view, next_url=None, next_field='next'):
    '''
    Creates a URL for redirecting to a login page. If only `login_view` is
    provided, this will just return the URL for it. If `next_url` is provided,
    however, this will append a ``next=URL`` parameter to the query string
    so that the login view can redirect back to that URL.

    :param login_view: The name of the login view. (Alternately, the actual
                       URL to the login view.)
    :type login_view: str
    :param next_url: The URL to give the login view for redirection.
    :type next_url: str
    :param next_field: What field to store the next URL in. (It defaults to
                       ``next``.)
    :type next_field: str
    '''
    if login_view.startswith(('https://', 'http://', '/')):
        base = login_view
    else:
        base = url_for(login_view)

    if next_url is None:
        return base

    parts = list(urlparse(base))
    md = url_decode(parts[4])
    md[next_field] = make_next_param(base, next_url)
    parts[4] = url_encode(md, sort=True)
    return urlunparse(parts) 
Example 25
Project: flasky   Author: RoseOu   File: pkg_resources.py    MIT License 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 26
Project: ANN   Author: waynezv   File: spider.py    MIT License 5 votes vote down vote up
def start_filt(self, url):
        if url:
            parsed_normal_url = urlparse.urlparse(url)
            parsed_origin_url = urlparse.urlparse(config.origin_url)
            netloc_origin = parsed_origin_url[1]
            if not parsed_normal_url[0]:
                url = urlparse.urlunparse((parsed_origin_url[0], netloc_origin, parsed_normal_url[2],\
                                          parsed_normal_url[3] ,parsed_normal_url[4] ,parsed_normal_url[5] ))
            parsed_normal_url = urlparse.urlparse(url)
            netloc_normal = parsed_normal_url[1]
            position = netloc_origin.find('.')
            domain = netloc_origin[:position]
            if domain in ["www"]:
                sub_domain = netloc_origin[position:]
            else:
                sub_domain = netloc_origin

            if self.is_allow_domain(netloc_normal, sub_domain):
                if not self.is_duplicate(url):
                    path = parsed_normal_url[2]
                    file_ext = path.split('.')[-1]
                    if file_ext in config.white_ext:
                        self.store_picture(url, file_ext)
                    else:
                        self.task.to_crawl.append(url)
                else:
                    #print "duplicate"
                    pass
            else:
                #print "external"
                pass 
Example 27
Project: openhatch   Author: campbe13   File: models.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def get_query_url(self):
        qs = [
            ('ws.op', 'searchTasks'),
            ('created_since', self.last_polled.isoformat())
        ]
        qs = urllib.urlencode(qs)
        parts = (
            'https',
            'api.launchpad.net',
            urlparse.urljoin('/1.0/', self.tracker.launchpad_name),
            '',
            qs,
            '')
        url = urlparse.urlunparse(parts)
        return url 
Example 28
Project: openhatch   Author: campbe13   File: request.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def request_httprepr(request):
    """Return the raw HTTP representation (as string) of the given request.
    This is provided only for reference since it's not the actual stream of
    bytes that will be send when performing the request (that's controlled
    by Twisted).
    """
    parsed = urlparse_cached(request)
    path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
    s  = "%s %s HTTP/1.1\r\n" % (request.method, path)
    s += "Host: %s\r\n" % parsed.hostname
    if request.headers:
        s += request.headers.to_string() + "\r\n"
    s += "\r\n"
    s += request.body
    return s 
Example 29
Project: openhatch   Author: campbe13   File: ClientForm.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def ParseResponseEx(response,
                    select_default=False,
                    form_parser_class=FormParser,
                    request_class=urllib2.Request,
                    entitydefs=None,
                    encoding=DEFAULT_ENCODING,

                    # private
                    _urljoin=urlparse.urljoin,
                    _urlparse=urlparse.urlparse,
                    _urlunparse=urlparse.urlunparse,
                    ):
    """Identical to ParseResponse, except that:

    1. The returned list contains an extra item.  The first form in the list
    contains all controls not contained in any FORM element.

    2. The arguments ignore_errors and backwards_compat have been removed.

    3. Backwards-compatibility mode (backwards_compat=True) is not available.
    """
    return _ParseFileEx(response, response.geturl(),
                        select_default,
                        False,
                        form_parser_class,
                        request_class,
                        entitydefs,
                        False,
                        encoding,
                        _urljoin=_urljoin,
                        _urlparse=_urlparse,
                        _urlunparse=_urlunparse,
                        ) 
Example 30
Project: openhatch   Author: campbe13   File: ClientForm.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def ParseFileEx(file, base_uri,
                select_default=False,
                form_parser_class=FormParser,
                request_class=urllib2.Request,
                entitydefs=None,
                encoding=DEFAULT_ENCODING,

                # private
                _urljoin=urlparse.urljoin,
                _urlparse=urlparse.urlparse,
                _urlunparse=urlparse.urlunparse,
                ):
    """Identical to ParseFile, except that:

    1. The returned list contains an extra item.  The first form in the list
    contains all controls not contained in any FORM element.

    2. The arguments ignore_errors and backwards_compat have been removed.

    3. Backwards-compatibility mode (backwards_compat=True) is not available.
    """
    return _ParseFileEx(file, base_uri,
                        select_default,
                        False,
                        form_parser_class,
                        request_class,
                        entitydefs,
                        False,
                        encoding,
                        _urljoin=_urljoin,
                        _urlparse=_urlparse,
                        _urlunparse=_urlunparse,
                        ) 
Example 31
Project: openhatch   Author: campbe13   File: ClientForm.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def __init__(self, action, method="GET",
                 enctype="application/x-www-form-urlencoded",
                 name=None, attrs=None,
                 request_class=urllib2.Request,
                 forms=None, labels=None, id_to_labels=None,
                 backwards_compat=True):
        """
        In the usual case, use ParseResponse (or ParseFile) to create new
        HTMLForm objects.

        action: full (absolute URI) form action
        method: "GET" or "POST"
        enctype: form transfer encoding MIME type
        name: name of form
        attrs: dictionary mapping original HTML form attributes to their values

        """
        self.action = action
        self.method = method
        self.enctype = enctype
        self.name = name
        if attrs is not None:
            self.attrs = attrs.copy()
        else:
            self.attrs = {}
        self.controls = []
        self._request_class = request_class

        # these attributes are used by zope.testbrowser
        self._forms = forms  # this is a semi-public API!
        self._labels = labels  # this is a semi-public API!
        self._id_to_labels = id_to_labels  # this is a semi-public API!

        self.backwards_compat = backwards_compat  # note __setattr__

        self._urlunparse = urlparse.urlunparse
        self._urlparse = urlparse.urlparse 
Example 32
Project: openhatch   Author: campbe13   File: ClientForm.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def click_request_data(self,
                           name=None, type=None, id=None,
                           nr=0, coord=(1,1),
                           request_class=urllib2.Request,
                           label=None):
        """As for click method, but return a tuple (url, data, headers).

        You can use this data to send a request to the server.  This is useful
        if you're using httplib or urllib rather than urllib2.  Otherwise, use
        the click method.

        # Untested.  Have to subclass to add headers, I think -- so use urllib2
        # instead!
        import urllib
        url, data, hdrs = form.click_request_data()
        r = urllib.urlopen(url, data)

        # Untested.  I don't know of any reason to use httplib -- you can get
        # just as much control with urllib2.
        import httplib, urlparse
        url, data, hdrs = form.click_request_data()
        tup = urlparse(url)
        host, path = tup[1], urlparse.urlunparse((None, None)+tup[2:])
        conn = httplib.HTTPConnection(host)
        if data:
            httplib.request("POST", path, data, hdrs)
        else:
            httplib.request("GET", path, headers=hdrs)
        r = conn.getresponse()

        """
        return self._click(name, type, id, label, nr, coord, "request_data",
                           self._request_class) 
Example 33
Project: openhatch   Author: campbe13   File: httpproxy.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _get_proxy(self, url, orig_type):
        proxy_type, user, password, hostport = _parse_proxy(url)
        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))

        if user and password:
            user_pass = '%s:%s' % (unquote(user), unquote(password))
            creds = base64.b64encode(user_pass).strip()
        else:
            creds = None

        return creds, proxy_url 
Example 34
Project: openhatch   Author: campbe13   File: webclient.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _parsed_url_args(parsed):
    path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
    host = parsed.hostname
    port = parsed.port
    scheme = parsed.scheme
    netloc = parsed.netloc
    if port is None:
        port = 443 if scheme == 'https' else 80
    return scheme, netloc, host, port, path 
Example 35
Project: openhatch   Author: campbe13   File: feedparser.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _urljoin(base, uri):
    uri = _urifixer.sub(r'\1\3', uri)
    try:
        return urlparse.urljoin(base, uri)
    except:
        uri = urlparse.urlunparse([urllib.quote(part) for part in urlparse.urlparse(uri)])
        return urlparse.urljoin(base, uri) 
Example 36
Project: taxonwiki   Author: philiptzou   File: itis.py    MIT License 5 votes vote down vote up
def itis_connect():
    uri = urlparse.urlparse(app.config['DATABASE_URI'])
    uri = uri[0:2] + ('ITIS',) + uri[3:]
    uri = urlparse.urlunparse(uri)
    engine = sqlalchemy.engine.create_engine(uri, server_side_cursors=True)
    conn = engine.connect()
    try:
        yield conn
    finally:
        conn.close() 
Example 37
Project: Pancas   Author: Sup3Roque   File: f4mDownloader.py    GNU General Public License v2.0 5 votes vote down vote up
def join(base,url):
    join = urlparse.urljoin(base,url)
    url = urlparse.urlparse(join)
    path = posixpath.normpath(url[2])
    return urlparse.urlunparse(
        (url.scheme,url.netloc,path,url.params,url.query,url.fragment)
        ) 
Example 38
Project: Pancas   Author: Sup3Roque   File: f4mDownloader.py    GNU General Public License v2.0 5 votes vote down vote up
def join(base,url):
    join = urlparse.urljoin(base,url)
    url = urlparse.urlparse(join)
    path = posixpath.normpath(url[2])
    return urlparse.urlunparse(
        (url.scheme,url.netloc,path,url.params,url.query,url.fragment)
        ) 
Example 39
Project: chattR   Author: patrickstocklin   File: __init__.py    GNU General Public License v2.0 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 40
Project: chattR   Author: patrickstocklin   File: __init__.py    GNU General Public License v2.0 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 41
Project: Miscellaneous   Author: Shinpachi8   File: classSQL.py    Apache License 2.0 5 votes vote down vote up
def get_value(self, payload=None):
        # to check if the file upload, if uploaded, return directly
        self.to_check_list = []
        #self.aim_error_list = []
        if payload is None: payload=classSQL.payload
        if self.method == 'GET':
            _ = urlparse.urlparse(self.url)
            if not _.query:
                return
            values = dict(urlparse.parse_qsl(_.query))
            for p in payload:
                for param in values.keys():
                    temp_dict = copy.deepcopy(values)
                    temp_dict[param] =   p
                    temp_query = urllib.urlencode(temp_dict)
                    self.to_check_list.append(urlparse.urlunparse((_.scheme, _.netloc, _.path, _.params, temp_query, _.fragment)))
        elif self.method == 'POST':
            jsondata = False
            try:
                values = dict(urlparse.parse_qsl(self.data))
            except Exception as e:
                print repr(e)
                try:
                    values = json.loads(self.data)
                    jsondata = True
                except:
                    return
            for p in payload:
                for param in values.keys():
                    temp_dict = copy.deepcopy(values)
                    temp_dict[param] =  p
                    if jsondata:
                        self.to_check_list.append(json.dumps(temp_dict))
                    else:
                        temp_query = urllib.urlencode(temp_dict)
                        self.to_check_list.append(temp_query)


        #print "self.to_check_list = {}".format(self.to_check_list) 
Example 42
Project: Miscellaneous   Author: Shinpachi8   File: common.py    Apache License 2.0 5 votes vote down vote up
def url_string(self):
        data = (self.scheme, self.netloc, self.path, self.params, self.query, self.fragment)
        url = urlparse.urlunparse(data)
        self.final_url = url
        return url 
Example 43
Project: Miscellaneous   Author: Shinpachi8   File: UrlDeDuplicate.py    Apache License 2.0 5 votes vote down vote up
def get_pattern(self):
        query = self.get_query_pattern()
        path = self.get_path_pattern()
        netloc = self.parsed_url.netloc
        scheme =self.parsed_url.scheme

        return urlparse.urlunparse((scheme, netloc, path, '', query, '')) 
Example 44
Project: analyzer-website   Author: santiagolizardo   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def get_callback_url(self):
        if self.callback and self.verifier:
            # Append the oauth_verifier.
            parts = urlparse.urlparse(self.callback)
            scheme, netloc, path, params, query, fragment = parts[:6]
            if query:
                query = '%s&oauth_verifier=%s' % (query, self.verifier)
            else:
                query = 'oauth_verifier=%s' % self.verifier
            return urlparse.urlunparse((scheme, netloc, path, params,
                query, fragment))
        return self.callback 
Example 45
Project: analyzer-website   Author: santiagolizardo   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def to_url(self):
        """Serialize as a URL for a GET request."""
        base_url = urlparse.urlparse(self.url)
        try:
            query = base_url.query
        except AttributeError:
            # must be python <2.5
            query = base_url[4]
        query = parse_qs(query)
        for k, v in self.items():
            query.setdefault(k, []).append(v)
        
        try:
            scheme = base_url.scheme
            netloc = base_url.netloc
            path = base_url.path
            params = base_url.params
            fragment = base_url.fragment
        except AttributeError:
            # must be python <2.5
            scheme = base_url[0]
            netloc = base_url[1]
            path = base_url[2]
            params = base_url[3]
            fragment = base_url[5]
        
        url = (scheme, netloc, path, params,
               urllib.urlencode(query, True), fragment)
        return urlparse.urlunparse(url) 
Example 46
Project: Flask_Blog   Author: sugarguo   File: pkg_resources.py    GNU General Public License v3.0 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 47
Project: Flask_Blog   Author: sugarguo   File: flask_login.py    GNU General Public License v3.0 5 votes vote down vote up
def login_url(login_view, next_url=None, next_field='next'):
    '''
    Creates a URL for redirecting to a login page. If only `login_view` is
    provided, this will just return the URL for it. If `next_url` is provided,
    however, this will append a ``next=URL`` parameter to the query string
    so that the login view can redirect back to that URL.

    :param login_view: The name of the login view. (Alternately, the actual
                       URL to the login view.)
    :type login_view: str
    :param next_url: The URL to give the login view for redirection.
    :type next_url: str
    :param next_field: What field to store the next URL in. (It defaults to
                       ``next``.)
    :type next_field: str
    '''
    if login_view.startswith(('https://', 'http://', '/')):
        base = login_view
    else:
        base = url_for(login_view)

    if next_url is None:
        return base

    parts = list(urlparse(base))
    md = url_decode(parts[4])
    md[next_field] = make_next_param(base, next_url)
    parts[4] = url_encode(md, sort=True)
    return urlunparse(parts) 
Example 48
Project: Flask_Blog   Author: sugarguo   File: pkg_resources.py    GNU General Public License v3.0 5 votes vote down vote up
def _remove_md5_fragment(location):
    if not location:
        return ''
    parsed = urlparse(location)
    if parsed[-1].startswith('md5='):
        return urlunparse(parsed[:-1] + ('',))
    return location 
Example 49
Project: wpt-sync   Author: mozilla   File: bug.py    Mozilla Public License 2.0 5 votes vote down vote up
def bz_url_from_api_url(api_url):
    if api_url is None:
        return None
    parts = urlparse.urlparse(api_url)
    bz_url = (parts.scheme, parts.netloc, "", "", "", "")
    return urlparse.urlunparse(bz_url) 
Example 50
Project: plugin.audio.subsonic   Author: basilfx   File: addon.py    MIT License 5 votes vote down vote up
def build_url(self, query):
        """
        Create URL for page.
        """

        parts = list(urlparse.urlparse(self.addon_url))
        parts[4] = urllib.urlencode(query)

        return urlparse.urlunparse(parts)