Python urllib.parse.urljoin() Examples

def create_new_user(requests_session, server, headers):
    # Get a list of groups and grab the ID of the first one
    response = requests_session.get(urljoin(server, "/api/v3/group/"), headers=headers)
    group_id = response.json()["results"][0]["id"]
    group_name = response.json()["results"][0]["name"]
    fake = Faker()

    # Create a user, assigning them to the group we obtained
    user_data = {
        "first_name": fake.first_name(),
        "last_name": fake.last_name(),
        "email": fake.company_email(),
        "groups": {"id": group_id, "name": group_name},
    }
    response = requests_session.post(
        urljoin(server, "/api/v3/user/"), headers=headers, data=user_data
    )

    return {
        "id": response.json()["id"],
        "first_name": response.json()["first_name"],
        "last_name": response.json()["last_name"],
        "group_id": group_id,
    } 

def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def maybe_download_mnist_file(file_name, datadir=None, force=False):
    try:
        from urllib.parse import urljoin
        from urllib.request import urlretrieve
    except ImportError:
        from urlparse import urljoin
        from urllib import urlretrieve

    if not datadir:
        datadir = tempfile.gettempdir()
    dest_file = os.path.join(datadir, file_name)

    if force or not os.path.isfile(file_name):
        url = urljoin('http://yann.lecun.com/exdb/mnist/', file_name)
        urlretrieve(url, dest_file)
    return dest_file 

def instance_endpoint_by_id(self, id=None):
        """
        Returns the endpoint designating some instantiated API resource of the
        same kind. If no `id` is provided, will use the `id` of the currently
        instantiated resource. If this is called from a static object, then
        returns `None`.
        """
        _id = self._get_id(id=id)

        if _id:
            # CASE 1: The class end point might have a formatting parameter
            # NOTE: This is for the weird case of submissions of an assignment
            try:
                tmp = self.class_endpoint.format(_id)
                if tmp != self.class_endpoint:
                    return tmp
            except IndexError: # means formatting didn't work
                pass

            # CASE 2: The class end point has not formatting parameter
            # NOTE: Trailing slash important (API bug)
            return urljoin(self.class_endpoint, "{}/".format(_id)) 

def new_recipe(requests_session, action_id, server, headers):
    urllib3.disable_warnings()

    # Create a recipe
    recipe_data = {
        "action_id": action_id,
        "arguments": '{"learnMoreMessage":"This field may not be blank.","learnMoreUrl":"This field may not be blank.","message":"This field may not be blank.","postAnswerUrl":"This field may not be blank.","surveyId":"'
        + str(uuid.uuid4())
        + '","thanksMessage":"This field may not be blank."}',
        "name": "test recipe",
        "extra_filter_expression": "counter == 0",
        "enabled": "false",
    }

    response = requests_session.post(
        urljoin(server, "/api/v3/recipe/"), data=recipe_data, headers=headers
    )
    data = response.json()
    return {"id": data["id"], "latest_revision_id": data["latest_revision"]["id"]} 

def test_static_cache_headers(conf, requests_session):
    """Test that all scripts included from self-repair have long lived cache headers"""
    req = requests_session.get(conf.getoption("server") + "/en-US/repair")
    req.raise_for_status()
    document = html5lib.parse(req.content, treebuilder="dom")
    scripts = document.getElementsByTagName("script")
    for script in scripts:
        src = script.getAttribute("src")
        url = urljoin(conf.getoption("server"), src)
        script_req = requests_session.get(url)
        script_req.raise_for_status()
        cache_control = parse_cache_control(script_req.headers["cache-control"])
        assert cache_control["public"], f"Cache-control: public for {url}"
        ONE_YEAR = 31_536_000
        assert cache_control["max-age"] >= ONE_YEAR, f"Cache-control: max-age > 1 year for {url}"
        assert cache_control["immutable"], f"Cache-control: immutable for {url}" 

def as_requests_kwargs(self, base_url: Optional[str] = None) -> Dict[str, Any]:
        """Convert the case into a dictionary acceptable by requests."""
        base_url = self._get_base_url(base_url)
        formatted_path = self.formatted_path.lstrip("/")  # pragma: no mutate
        url = urljoin(base_url + "/", formatted_path)
        # Form data and body are mutually exclusive
        extra: Dict[str, Optional[Body]]
        if self.form_data:
            extra = {"files": self.form_data}
        elif is_multipart(self.body):
            extra = {"data": self.body}
        else:
            extra = {"json": self.body}
        return {
            "method": self.method,
            "url": url,
            "cookies": self.cookies,
            "headers": self.headers,
            "params": self.query,
            **extra,
        } 

def __init__(self, account, log_level=None, api_base_url=GODADDY_API_BASE_URL, api_version=GODADDY_API_VERSION):
        """Create a new `godaddypy.Client` object

        :type account: godaddypy.Account
        :param account: The godaddypy.Account object to create auth headers with.
        """

        # Logging setup
        self.logger = logging.getLogger('GoDaddyPy.Client')
        # Explicit override of logging level
        if log_level is not None:
            self.logger.setLevel(log_level)

        # Templates
        self.API_TEMPLATE = urljoin(api_base_url, api_version)
        self.DOMAINS = '/domains'
        self.DOMAIN_INFO = '/domains/{domain}'
        self.RECORDS = '/domains/{domain}/records'
        self.RECORDS_TYPE = '/domains/{domain}/records/{type}'
        self.RECORDS_TYPE_NAME = '/domains/{domain}/records/{type}/{name}'

        self.account = account 

def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)#获取到
        print('fetched %s' % url)


        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 

def __init__(self, board_url_or_id):
        board_id = str(board_url_or_id)
        self.fetcher = HuaBanFetcher()
        if "http" in board_id:
            board_id = re.findall(r'boards/(\d+)/', board_id)[0]
        self.id = board_id
        path = "/boards/{board_id}/".format(
            board_id=board_id,
        )
        self.base_url = urljoin(BASE_URL, path)
        self.further_pin_url_tpl = urljoin(
            self.base_url,
            "?{random_string}"
            "&max={pin_id}"
            "&limit=20"
            "&wfl=1"
        )

        # uninitialized properties
        self.pin_count = None
        self.title = None
        self.description = None
        self._pins = []
        self._init_board() 

def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)
        print('fetched %s' % url)

        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 

def _join_url(self, path):
        if path.startswith('/'):
            path = path[1:]
        return urljoin(self.BASE_URL, path) 

def path2url(path):
    return urljoin(
        'file:', pathname2url(os.path.abspath(path))) 

def _delete_pagefile(request, course_slug, page_label, kind):
    """
    Delete page/file
    """
    with django.db.transaction.atomic():
        offering = get_object_or_404(CourseOffering, slug=course_slug)
        page = get_object_or_404(Page, offering=offering, label=page_label)
        version = page.current_version()
        member = _check_allowed(request, offering, page.can_write, page.editdate())
        if not member:
            return ForbiddenResponse(request, 'Not allowed to edit this '+kind+'.')
        can_create = member.role in MEMBER_ROLES[offering.page_creators()]
        if not can_create:
            return ForbiddenResponse(request, 'Not allowed to delete pages in for this offering (must have page-creator permission).')

        from django.core.validators import URLValidator
        from django.core.exceptions import ValidationError
        val = URLValidator()

        redirect = request.POST.get('redirect', 'Index')
        #  Technically, the empty string is a valid value for the redirect because the field allows blanks.
        #  We want to avoid that when deleting a page so we need an extra check here.
        if not redirect:
            redirect = 'Index'
        url = request.build_absolute_uri(urljoin(page.get_absolute_url(), redirect))

        try:
            val(url)
        except ValidationError:
            messages.error(request, "Bad redirect URL entered. Not deleted.")
            return HttpResponseRedirect(reverse('offering:pages:edit_page', kwargs={'course_slug': course_slug, 'page_label': page.label}))

        redir_version = PageVersion(page=page, title=version.title, redirect=redirect,
                                    editor=member, comment='automatically generated on deletion')
        redir_version.set_redirect_reason('delete')
        redir_version.save()

        messages.success(request, "Page deleted and will redirect to this location.")
        return HttpResponseRedirect(urljoin(page.get_absolute_url(), redirect)) 

def _get_from_json(self, *, name, version):
        url = urljoin(self.url, posixpath.join(name, str(version), 'json'))
        async with aiohttp_session(auth=self.auth) as session:
            async with session.get(url) as response:
                if response.status == 404:
                    raise PackageNotFoundError(package=name, url=url)
                response.raise_for_status()
                response = await response.json()
        dist = response['info']['requires_dist'] or []
        if dist:
            return dist

        # If no requires_dist then package metadata can be broken.
        # Let's check distribution files.
        return await self._get_from_files(response['urls']) 

def _has_api(url: str) -> bool:
    if urlparse(url).hostname in ('pypi.org', 'python.org', 'test.pypi.org'):
        return True
    full_url = urljoin(url, 'dephell/json/')
    try:
        response = requests.head(full_url)
    except (SSLError, ConnectionError):
        return False
    return response.status_code < 400 

def _get_links(self, name: str) -> List[Dict[str, str]]:
        cache = JSONCache(
            'warehouse-simple', urlparse(self.url).hostname, 'links', name,
            ttl=config['cache']['ttl'],
        )
        links = cache.load()
        if links is not None:
            yield from links
            return

        dep_url = posixpath.join(self.url, quote(name)) + '/'
        with requests_session() as session:
            logger.debug('getting dep info from simple repo', extra=dict(url=dep_url))
            response = session.get(dep_url, auth=self.auth)
        if response.status_code == 404:
            raise PackageNotFoundError(package=name, url=dep_url)
        response.raise_for_status()
        document = html5lib.parse(response.text, namespaceHTMLElements=False)

        links = []
        for tag in document.findall('.//a'):
            link = tag.get('href')
            if not link:
                continue
            parsed = urlparse(link)
            if not parsed.path.endswith(ARCHIVE_EXTENSIONS):
                continue

            python = tag.get('data-requires-python')
            fragment = parse_qs(parsed.fragment)
            link = dict(
                url=urljoin(dep_url, link),
                name=parsed.path.strip('/').split('/')[-1],
                python=html.unescape(python) if python else '*',
                digest=fragment['sha256'][0] if 'sha256' in fragment else None,
            )
            links.append(link)
            yield link

        cache.dump(links)
        return links 

def create_approval_request(requests_session, server, latest_revision_id, headers):
    return requests_session.post(
        urljoin(server, "/api/v3/recipe_revision/{}/request_approval/".format(latest_revision_id)),
        headers=headers,
    ) 

def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.chat_url = urljoin(self.BASE_URL, 'chat.postMessage')
        self.channels = [ch.strip() for ch in SLACK_CHANNEL.split(',')]
        self.channels_unique_settings = SLACK_CHANNELS_UNIQUE_SETTINGS
        self.bot_name = 'zoom2youtube'
        self.token = SLACK_TOKEN 

def get(self, request, *args, **kwargs):
        ret = {}

        # Get the version namespace
        namespace = getattr(request.resolver_match, "namespace", "")
        for ns in namespace.split(":"):
            if re.match(r"v[0-9]+", ns, re.I):
                namespace = ns
                break

        if self.urlconf:
            urlconf = self.urlconf
        else:
            urlconf = import_module(settings.ROOT_URLCONF)
        endpoints = get_api_endpoints(urlconf.urlpatterns)

        for endpoint in sorted(endpoints, key=lambda e: e["pattern"].name):
            name = endpoint["pattern"].name
            if endpoint["method"] == "GET" and namespace in endpoint["namespace"].split(":"):
                allow_cdn = getattr(endpoint["pattern"], "allow_cdn", True)

                if not allow_cdn and settings.APP_SERVER_URL:
                    base = settings.APP_SERVER_URL
                else:
                    base = request.build_absolute_uri()

                try:
                    full_name = (
                        f'{endpoint["namespace"]}:{name}' if endpoint["namespace"] else name
                    )
                    path = reverse(full_name, *args, **kwargs)
                except NoReverseMatch:
                    continue

                full_url = urljoin(base, path)
                ret[name] = full_url

        return Response(ret) 

def test_recipe_revision_list(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/recipe_revision/"))
    assert response.status_code != 404
    assert_valid_schema(response.json()) 

def test_recipe_history(conf, requests_session):
    # Get the ID of a random recipe and grab it's history
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/recipe/"))
    data = response.json()

    if len(data["results"]) == 0:
        pytest.skip("Could not find any recipe history")

    idx = randint(0, len(data["results"]) - 1)
    id = data["results"][idx]["id"]
    response = requests_session.get(
        urljoin(conf.getoption("server"), "/api/v3/recipe/{}/history/".format(id))
    )
    assert response.status_code != 404
    assert_valid_schema(response.json()) 

def test_filters_list(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/filters/"))
    assert response.status_code != 404
    assert_valid_schema(response.json()) 

def test_extension_list(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/extension/"))
    assert response.status_code != 404
    assert_valid_schema(response.json()) 

def test_extension_read(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/extension/"))
    assert response.status_code != 404
    details = response.json()

    # Get the ID of the first record in the list and read it
    if len(details["results"]) == 0:
        pytest.skip("No extensions results were found")

    extension_id = details["results"][0]["id"]
    response = requests_session.get(
        urljoin(conf.getoption("server"), "/api/v3/extension/{}".format(extension_id))
    )
    assert_valid_schema(response.json()) 

def test_recipe_list(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3/recipe/"))
    assert response.status_code != 404
    assert_valid_schema(response.json()) 

def enable_recipe(requests_session, server, recipe_id, headers):
    return requests_session.post(
        urljoin(server, "/api/v3/recipe/{}/enable/".format(recipe_id)), headers=headers
    ) 

def test_list(conf, requests_session):
    response = requests_session.get(urljoin(conf.getoption("server"), "/api/v3"))
    assert response.status_code != 404
    assert_valid_schema(response.json())