Python urllib.parse.urljoin() Examples

The following are 30 code examples for showing how to use urllib.parse.urljoin(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module urllib.parse , or try the search function .

Example 1
Project: normandy   Author: mozilla   File: helpers.py    License: Mozilla Public License 2.0 7 votes vote down vote up
def create_new_user(requests_session, server, headers):
    # Get a list of groups and grab the ID of the first one
    response = requests_session.get(urljoin(server, "/api/v3/group/"), headers=headers)
    group_id = response.json()["results"][0]["id"]
    group_name = response.json()["results"][0]["name"]
    fake = Faker()

    # Create a user, assigning them to the group we obtained
    user_data = {
        "first_name": fake.first_name(),
        "last_name": fake.last_name(),
        "email": fake.company_email(),
        "groups": {"id": group_id, "name": group_name},
    }
    response = requests_session.post(
        urljoin(server, "/api/v3/user/"), headers=headers, data=user_data
    )

    return {
        "id": response.json()["id"],
        "first_name": response.json()["first_name"],
        "last_name": response.json()["last_name"],
        "group_id": group_id,
    } 
Example 2
Project: neural-fingerprinting   Author: StephanZheng   File: utils_mnist.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def maybe_download_mnist_file(file_name, datadir=None, force=False):
    try:
        from urllib.parse import urljoin
        from urllib.request import urlretrieve
    except ImportError:
        from urlparse import urljoin
        from urllib import urlretrieve

    if not datadir:
        datadir = tempfile.gettempdir()
    dest_file = os.path.join(datadir, file_name)

    if force or not os.path.isfile(file_name):
        url = urljoin('http://yann.lecun.com/exdb/mnist/', file_name)
        urlretrieve(url, dest_file)
    return dest_file 
Example 3
Project: codepost-python   Author: codepost-io   File: api_resource.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def instance_endpoint_by_id(self, id=None):
        """
        Returns the endpoint designating some instantiated API resource of the
        same kind. If no `id` is provided, will use the `id` of the currently
        instantiated resource. If this is called from a static object, then
        returns `None`.
        """
        _id = self._get_id(id=id)

        if _id:
            # CASE 1: The class end point might have a formatting parameter
            # NOTE: This is for the weird case of submissions of an assignment
            try:
                tmp = self.class_endpoint.format(_id)
                if tmp != self.class_endpoint:
                    return tmp
            except IndexError: # means formatting didn't work
                pass

            # CASE 2: The class end point has not formatting parameter
            # NOTE: Trailing slash important (API bug)
            return urljoin(self.class_endpoint, "{}/".format(_id)) 
Example 4
Project: PickTrue   Author: winkidney   File: huaban.py    License: MIT License 6 votes vote down vote up
def __init__(self, board_url_or_id):
        board_id = str(board_url_or_id)
        self.fetcher = HuaBanFetcher()
        if "http" in board_id:
            board_id = re.findall(r'boards/(\d+)/', board_id)[0]
        self.id = board_id
        path = "/boards/{board_id}/".format(
            board_id=board_id,
        )
        self.base_url = urljoin(BASE_URL, path)
        self.further_pin_url_tpl = urljoin(
            self.base_url,
            "?{random_string}"
            "&max={pin_id}"
            "&limit=20"
            "&wfl=1"
        )

        # uninitialized properties
        self.pin_count = None
        self.title = None
        self.description = None
        self._pins = []
        self._init_board() 
Example 5
Project: controller   Author: deis   File: __init__.py    License: MIT License 6 votes vote down vote up
def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 6
Project: controller   Author: deis   File: __init__.py    License: MIT License 6 votes vote down vote up
def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 7
Project: controller   Author: deis   File: __init__.py    License: MIT License 6 votes vote down vote up
def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 8
Project: tornado-zh   Author: tao12345666333   File: webspider.py    License: MIT License 6 votes vote down vote up
def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)
        print('fetched %s' % url)

        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 
Example 9
def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)#获取到
        print('fetched %s' % url)


        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 
Example 10
Project: normandy   Author: mozilla   File: test_performance.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def test_static_cache_headers(conf, requests_session):
    """Test that all scripts included from self-repair have long lived cache headers"""
    req = requests_session.get(conf.getoption("server") + "/en-US/repair")
    req.raise_for_status()
    document = html5lib.parse(req.content, treebuilder="dom")
    scripts = document.getElementsByTagName("script")
    for script in scripts:
        src = script.getAttribute("src")
        url = urljoin(conf.getoption("server"), src)
        script_req = requests_session.get(url)
        script_req.raise_for_status()
        cache_control = parse_cache_control(script_req.headers["cache-control"])
        assert cache_control["public"], f"Cache-control: public for {url}"
        ONE_YEAR = 31_536_000
        assert cache_control["max-age"] >= ONE_YEAR, f"Cache-control: max-age > 1 year for {url}"
        assert cache_control["immutable"], f"Cache-control: immutable for {url}" 
Example 11
Project: normandy   Author: mozilla   File: helpers.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def new_recipe(requests_session, action_id, server, headers):
    urllib3.disable_warnings()

    # Create a recipe
    recipe_data = {
        "action_id": action_id,
        "arguments": '{"learnMoreMessage":"This field may not be blank.","learnMoreUrl":"This field may not be blank.","message":"This field may not be blank.","postAnswerUrl":"This field may not be blank.","surveyId":"'
        + str(uuid.uuid4())
        + '","thanksMessage":"This field may not be blank."}',
        "name": "test recipe",
        "extra_filter_expression": "counter == 0",
        "enabled": "false",
    }

    response = requests_session.post(
        urljoin(server, "/api/v3/recipe/"), data=recipe_data, headers=headers
    )
    data = response.json()
    return {"id": data["id"], "latest_revision_id": data["latest_revision"]["id"]} 
Example 12
Project: godaddypy   Author: eXamadeus   File: client.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def __init__(self, account, log_level=None, api_base_url=GODADDY_API_BASE_URL, api_version=GODADDY_API_VERSION):
        """Create a new `godaddypy.Client` object

        :type account: godaddypy.Account
        :param account: The godaddypy.Account object to create auth headers with.
        """

        # Logging setup
        self.logger = logging.getLogger('GoDaddyPy.Client')
        # Explicit override of logging level
        if log_level is not None:
            self.logger.setLevel(log_level)

        # Templates
        self.API_TEMPLATE = urljoin(api_base_url, api_version)
        self.DOMAINS = '/domains'
        self.DOMAIN_INFO = '/domains/{domain}'
        self.RECORDS = '/domains/{domain}/records'
        self.RECORDS_TYPE = '/domains/{domain}/records/{type}'
        self.RECORDS_TYPE_NAME = '/domains/{domain}/records/{type}/{name}'

        self.account = account 
Example 13
Project: schemathesis   Author: kiwicom   File: models.py    License: MIT License 6 votes vote down vote up
def as_requests_kwargs(self, base_url: Optional[str] = None) -> Dict[str, Any]:
        """Convert the case into a dictionary acceptable by requests."""
        base_url = self._get_base_url(base_url)
        formatted_path = self.formatted_path.lstrip("/")  # pragma: no mutate
        url = urljoin(base_url + "/", formatted_path)
        # Form data and body are mutually exclusive
        extra: Dict[str, Optional[Body]]
        if self.form_data:
            extra = {"files": self.form_data}
        elif is_multipart(self.body):
            extra = {"data": self.body}
        else:
            extra = {"json": self.body}
        return {
            "method": self.method,
            "url": url,
            "cookies": self.cookies,
            "headers": self.headers,
            "params": self.query,
            **extra,
        } 
Example 14
Project: zmirror   Author: aploium   File: utils.py    License: MIT License 5 votes vote down vote up
def extract_real_url_from_embedded_url(embedded_url):
    """
    将 embed_real_url_to_embedded_url() 编码后的url转换为原来的带有参数的url
    `cdn_redirect_encode_query_str_into_url`设置依赖于本函数, 详细说明请看配置文件中这个参数的部分

    eg: https://cdn.domain.com/a.php_zm24_.cT1zb21ldGhpbmc=._zm24_.css
        ---> https://foo.com/a.php?q=something (assume it returns an css) (base64 only)
    eg2: https://cdn.domain.com/a/b/_zm24_.bG92ZT1saXZl._zm24_.jpg
        ---> https://foo.com/a/b/?love=live (assume it returns an jpg) (base64 only)
    eg3: https://cdn.domain.com/a/b/_zm24z_.[some long long base64 encoded string]._zm24_.jpg
        ---> https://foo.com/a/b/?love=live[and a long long query string] (assume it returns an jpg) (gzip + base64)
    eg4:https://cdn.domain.com/a  (no change)
        ---> (no query string): https://foo.com/a (assume it returns an png) (no change)
    :param embedded_url: 可能被编码的URL
    :return: 如果传入的是编码后的URL, 则返回解码后的URL, 否则返回None
    :type embedded_url: str
    :rtype: Union[str, None]
    """
    if '._' + cdn_url_query_encode_salt + '_.' not in embedded_url[-15:]:  # check url mark
        return None
    m = regex_extract_base64_from_embedded_url.search(embedded_url)
    b64 = get_group('b64', m)

    # 'https://cdn.domain.com/a.php_zm24_.cT1zb21ldGhpbmc=._zm24_.css'
    # real_request_url_no_query ---> 'https://cdn.domain.com/a.php'
    real_request_url_no_query = embedded_url[:m.span()[0]]

    query_string_byte = base64.urlsafe_b64decode(b64)
    is_gzipped = get_group('gzip', m)
    if is_gzipped:
        query_string_byte = zlib.decompress(query_string_byte)
    query_string = query_string_byte.decode(encoding='utf-8')

    result = urljoin(real_request_url_no_query, '?' + query_string)
    # dbgprint('extract:', embedded_url, 'to', result)
    return result 
Example 15
Project: zmirror   Author: aploium   File: zmirror.py    License: MIT License 5 votes vote down vote up
def assemble_remote_url():
    """
    组装目标服务器URL, 即生成 parse.remote_url 的值
    :rtype: str
    """
    if parse.is_external_domain:
        # 请求的是外部域名 (external domains)
        scheme = 'https://' if parse.is_https else 'http://'
        return urljoin(scheme + parse.remote_domain, parse.remote_path_query)
    else:
        # 请求的是主域名及可以被当做(alias)主域名的域名
        return urljoin(target_scheme + target_domain, parse.remote_path_query) 
Example 16
Project: zmirror   Author: aploium   File: base_class.py    License: MIT License 5 votes vote down vote up
def url(self, path):
        domain = getattr(self.C, "my_host_name", "127.0.0.1")
        scheme = getattr(self.C, "my_host_scheme", "http://")
        return urljoin(scheme + domain, path) 
Example 17
Project: codepost-python   Author: codepost-io   File: api_requestor.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def _request(self, endpoint, method="get", **kwargs):
        kws = {}

        if kwargs and isinstance(kwargs, dict) and len(kwargs) > 0:
            kws.update(_copy.deepcopy(kwargs))

        kws["method"] = method

        # Configure API key

        api_key = kws.get("api_key", self.api_key)
        if "api_key" in kws:
            del kws["api_key"]

        # Compile headers

        default_headers = self._build_headers(
            api_key=api_key,
            **kws
        )
        kws["headers"] = kws.get("headers", dict())
        kws["headers"].update(default_headers)

        # Provide POSTed data as a JSON string

        if "application/json" in kws["headers"].get("Content-Type", ""):
            if "data" in kws and not isinstance(type(kws["data"]), str):
                kws["data"] = _json.dumps(kws["data"])
        
        ret = self._client.request(
            url=urljoin(self._base_url, endpoint),
            **kws
        )

        # Handle error codes here
        if not ret.status_code == 200:
            self._handle_request_error(response=ret)

        return ret

# ============================================================================= 
Example 18
Project: fishroom   Author: tuna   File: handlers.py    License: GNU General Public License v3.0 5 votes vote down vote up
def get(self):
        url = "log/{room}/today".format(
            room=config["chatlog"]["default_channel"]
        )
        self.redirect(urljoin(config["baseurl"] + "/", url)) 
Example 19
Project: python-rest-api   Author: messagebird   File: http_client.py    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
def request(self, path, method='GET', params=None, format=ResponseFormat.text):
        """Builds a request and gets a response."""
        if params is None:
            params = {}
        url = urljoin(self.endpoint, path)
        headers = {
            'Accept': 'application/json',
            'Authorization': 'AccessKey ' + self.access_key,
            'User-Agent': self.user_agent,
            'Content-Type': 'application/json'
        }

        method_switcher = {
            'DELETE': lambda: requests.delete(url, verify=True, headers=headers, data=json_serialize(params)),
            'GET': lambda: requests.get(url, verify=True, headers=headers, params=params),
            'PATCH': lambda: requests.patch(url, verify=True, headers=headers, data=json_serialize(params)),
            'POST': lambda: requests.post(url, verify=True, headers=headers, data=json_serialize(params)),
            'PUT': lambda: requests.put(url, verify=True, headers=headers, data=json_serialize(params))
        }
        if method not in method_switcher:
            raise ValueError(str(method) + ' is not a supported HTTP method')

        response = method_switcher[method]()

        if response.status_code not in self.__supported_status_codes:
            response.raise_for_status()

        response_switcher = {
            ResponseFormat.text: response.text,
            ResponseFormat.binary: response.content
        }
        return response_switcher.get(format) 
Example 20
Project: PickTrue   Author: winkidney   File: artstation.py    License: MIT License 5 votes vote down vote up
def get_project_page_url(username, page=1):
    path = PROJECT_URL_TPL.format(
        username=username,
        page=page,
    )
    url = urljoin(BASE_URL, path)
    return url 
Example 21
Project: PickTrue   Author: winkidney   File: artstation.py    License: MIT License 5 votes vote down vote up
def get_project_albums_page_url(user_id, page=1):
    path = ALBUMS_URL_TPL.format(
        user_id=user_id,
        page=page,
    )
    url = urljoin(BASE_URL, path)
    return url 
Example 22
Project: PickTrue   Author: winkidney   File: artstation.py    License: MIT License 5 votes vote down vote up
def get_project_albums_details_page_url(username, album_id, page=1):
    path = ALBUM_CONTENT_URL_TPL.format(
        username=username,
        album_id=album_id,
        page=page,
    )
    url = urljoin(BASE_URL, path)
    return url 
Example 23
Project: lichess-bot   Author: ShailChoksi   File: lichess.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def api_get(self, path):
        url = urljoin(self.baseUrl, path)
        response = self.session.get(url, timeout=2)
        response.raise_for_status()
        return response.json() 
Example 24
Project: lichess-bot   Author: ShailChoksi   File: lichess.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def api_post(self, path, data=None):
        url = urljoin(self.baseUrl, path)
        response = self.session.post(url, data=data, timeout=2)
        response.raise_for_status()
        return response.json() 
Example 25
Project: lichess-bot   Author: ShailChoksi   File: lichess.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def get_event_stream(self):
        url = urljoin(self.baseUrl, ENDPOINTS["stream_event"])
        return requests.get(url, headers=self.header, stream=True) 
Example 26
Project: lichess-bot   Author: ShailChoksi   File: lichess.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def get_game_stream(self, game_id):
        url = urljoin(self.baseUrl, ENDPOINTS["stream"].format(game_id))
        return requests.get(url, headers=self.header, stream=True) 
Example 27
Project: lichess-bot   Author: ShailChoksi   File: model.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def url(self):
        return urljoin(self.base_url, "{}/{}".format(self.id, self.my_color)) 
Example 28
Project: pinnwand   Author: supakeen   File: api_deprecated.py    License: MIT License 5 votes vote down vote up
def post(self) -> None:
        lexer = self.get_body_argument("lexer")
        raw = self.get_body_argument("code", strip=False)
        expiry = self.get_body_argument("expiry")
        filename = self.get_body_argument("filename", None)

        if not raw.strip():
            log.info("APINew.post: a paste was submitted without content")
            raise tornado.web.HTTPError(400)

        if lexer not in utility.list_languages():
            log.info("APINew.post: a paste was submitted with an invalid lexer")
            raise tornado.web.HTTPError(400)

        if expiry not in utility.expiries:
            log.info(
                "APINew.post: a paste was submitted with an invalid expiry"
            )
            raise tornado.web.HTTPError(400)

        paste = database.Paste(
            utility.slug_create(), utility.expiries[expiry], "deprecated-api"
        )
        paste.files.append(database.File(paste.slug, raw, lexer, filename))

        with database.session() as session:
            session.add(paste)
            session.commit()

            req_url = self.request.full_url()
            location = paste.slug
            if filename:
                location += "#" + url_escape(filename)
            self.write(
                {
                    "paste_id": paste.slug,
                    "removal_id": paste.removal,
                    "paste_url": urljoin(req_url, f"/{location}"),
                    "raw_url": urljoin(req_url, f"/raw/{paste.files[0].slug}"),
                }
            ) 
Example 29
Project: controller   Author: deis   File: __init__.py    License: MIT License 5 votes vote down vote up
def http_get(self, path, params=None, **kwargs):
        """
        Make a GET request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.get(url, params=params, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving data from " \
                      "the Kubernetes API server. URL: {}, params: {}".format(url, params)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 30
Project: controller   Author: deis   File: __init__.py    License: MIT License 5 votes vote down vote up
def http_delete(self, path, **kwargs):
        """
        Make a DELETE request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.delete(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem deleting data from " \
                      "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response