Python os.PathLike() Examples

def compute_hashes(file: "PathLike[str]") -> Dict[str, str]:
    """Compute hashes for a given file/directory.

    :param file_: path-like object pointing to a file/directory.

    :returns: dictionary that contains hash types as keys and corresponding
        hashes as values. There is one entry in this dictionary for each hash
        type in StreamHasher.KNOWN_HASH_TYPES.
        If file_ points to a directory values are empty strings.
    """
    path = Path(file)
    if path.is_dir():
        return {hash_type: "" for hash_type in StreamHasher.KNOWN_HASH_TYPES}

    hasher = StreamHasher()
    with path.open("rb") as stream:
        hasher.compute(stream)
    return {
        hash_type: hasher.hexdigest(hash_type)
        for hash_type in StreamHasher.KNOWN_HASH_TYPES
    } 

def validate_url(wrapped, instance, args, kwargs):
    """Enforces argument named "url" to be relative path.

    Check that it is instance of str or os.PathLike and that it represents
    relative path.
    """
    try:
        # Use -1 since self is not included in the args.
        url = args[getfullargspec(wrapped).args.index("url") - 1]
    except IndexError:
        url = kwargs.get("url")
    if not isinstance(url, (str, PathLike)):
        raise TypeError("Argument 'url' must be a string or path-like object")
    if PurePath(url).is_absolute():
        raise ValueError("Argument 'url' must be a relative path")
    return wrapped(*args, **kwargs) 

def clean_file(file):
	"""Returns a tuple containing a ``file``-like object and a close indicator.

	This ensures the given file is opened and keeps track of files that should
	be closed after use (files that were not open prior to this function call).

	Raises
	------
	OSError : Accessing the given file path failed

	Parameters
	----------
	file : Union[str, bytes, os.PathLike, io.IOBase, int]
		A filepath or ``file``-like object that may or may not need to be
		opened
	"""
	if isinstance(file, int):
		return os.fdopen(file, 'rb', closefd=False), True
	elif not hasattr(file, 'read'):
		return open(convert_path(file), 'rb'), True
	else:
		return file, False 

def write_meta_header(filename: PathLike, meta_dict: Dict[str, Any]):
    """
    Write the MHD meta header file

    :param filename: file to write
    :param meta_dict: dictionary of meta data in MetaImage format
    """
    header = ''
    # do not use tags = meta_dict.keys() because the order of tags matters
    tags = ['ObjectType', 'NDims', 'BinaryData',
            'BinaryDataByteOrderMSB', 'CompressedData', 'CompressedDataSize',
            'TransformMatrix', 'Offset', 'CenterOfRotation',
            'AnatomicalOrientation', 'ElementSpacing',
            'DimSize', 'ElementNumberOfChannels', 'ElementType', 'ElementDataFile',
            'Comment', 'SeriesDescription', 'AcquisitionDate',
            'AcquisitionTime', 'StudyDate', 'StudyTime']
    for tag in tags:
        if tag in meta_dict.keys():
            header += '%s = %s\n' % (tag, meta_dict[tag])
    with open(filename, 'w') as f:
        f.write(header) 

def uncompress_file(filepath: str or os.PathLike, outpath='.'):
    """
    Unzip a file to the same location of filepath
    uses decompressing algorithm by file extension

    Args:
        filepath (str): path to file
        outpath (str): path to extract to
    """
    filepath = str(filepath)
    if filepath.endswith('.zip'):
        with zipfile.ZipFile(filepath) as z:
            z.extractall(outpath)
    elif filepath.endswith('.gz'):
        if os.path.isdir(outpath):
            raise ValueError('output path for gzip must be a file')
        with gzip.open(filepath, 'rb') as fp:
            file_content = fp.read()
        with open(outpath, 'wb') as fp:
            fp.write(file_content)
    else:
        raise ValueError('Unsupported archive provided. Method supports only .zip/.gz files.') 

def presigned_url(
        self,
        url: Union[str, PathLike],
        expiration: int = 10,
        force_download: bool = False,
    ) -> Optional[str]:
        """Create a presigned URL.

        The URL is used to obtain temporary access to the object ar the
        given URL using only returned URL.

        :param expiration: expiration time of the link (in seconds), default
            is 10 seconds.

        :param force_download: force download.

        :returns: URL that can be used to access object or None.
        """
        raise NotImplementedError 

def _update_timestamp(path: os.PathLike, set_new: bool) -> None:
    """
    Context manager to set the timestamp of the path to plus or
    minus a fixed delta, regardless of modifications within the context.

    if set_new is True, the delta is added. Otherwise, the delta is subtracted.
    """
    stats = os.stat(path)
    if set_new:
        new_timestamp = (stats.st_atime_ns + _TIMESTAMP_DELTA, stats.st_mtime_ns + _TIMESTAMP_DELTA)
    else:
        new_timestamp = (stats.st_atime_ns - _TIMESTAMP_DELTA, stats.st_mtime_ns - _TIMESTAMP_DELTA)
    try:
        yield
    finally:
        os.utime(path, ns=new_timestamp)


# Public Methods 

def load(source_dir: Union[os.PathLike, str]) -> Any:
    """
    Load an object from a directory, saved by
    ``gordo.serializer.pipeline_serializer.dump``

    This take a directory, which is either top-level, meaning it contains
    a sub directory in the naming scheme: "n_step=<int>-class=<path.to.Class>"
    or the aforementioned naming scheme directory directly. Will return that
    unsterilized object.


    Parameters
    ----------
    source_dir: Union[os.PathLike, str]
        Location of the top level dir the pipeline was saved

    Returns
    -------
    Union[GordoBase, Pipeline, BaseEstimator]
    """
    # This source dir should have a single pipeline entry directory.
    # may have been passed a top level dir, containing such an entry:
    with open(os.path.join(source_dir, "model.pkl"), "rb") as f:
        return pickle.load(f) 

def read_data_cache(
    cache_key: bytes,
    path: os.PathLike,
    *,
    pickled: bool=True,
    source_dir: Optional[pathlib.Path] = None,
) -> Any:
    if source_dir is None:
        source_dir = get_shared_data_dir_path()
    full_path = source_dir / path

    if full_path.exists():
        with open(full_path, 'rb') as f:
            src_hash = f.read(len(cache_key))
            if src_hash == cache_key:
                if pickled:
                    data = f.read()
                    try:
                        return pickle.loads(data)
                    except Exception:
                        logging.exception(f'could not unpickle {path}')
                else:
                    return f.read() 

def before_push(self, objects: List[dict], url: Union[str, PathLike]):
        """Perform pre-processing before push.

        :param objects: objects to transfer.

        :param url: URL that will be used by transfer.
        """ 

def __init__(
            self,
            pt_exec: Union[List[str], List[bytes]],
            state: Union[str, bytes, os.PathLike],
            *,
            exit_on_stdin_close: bool = True,
    ) -> None:
        """Create the adapter.

        Args:
            pt_exec: The pluggable transport command line to execute. This has
                to be a list of str / bytes, since
                :func:`asyncio.create_subprocess_exec` does not accept an
                entire command line as a string. On non-Windows platforms
                :func:`shlex.split` can be used to split a command line string
                into a list, while on Windows it's a bit more complicated.
            state: The state directory. This is a directory where the PT is
                allowed to store state. Either specify a path (which
                is not required to exist, in which case the PT will create
                the directory), or specify ``None`` to use a temporary
                directory created using :mod:`tempfile`.
            exit_on_stdin_close: Whether closing the PT's STDIN indicates the
                PT should gracefully exit.
        """
        if isinstance(pt_exec, (str, bytes)):
            self._pt_args = [pt_exec]
        else:
            self._pt_args = list(pt_exec)
        if state is not None:
            self._state = os.path.abspath(state)
        else:
            self._state = None
        self._exit_on_stdin_close = exit_on_stdin_close

        self._process: asyncio.subprocess.Process = None
        self._stdout_task: asyncio.Task = None
        self._ready = asyncio.Future()
        self._accepted_version: str = None
        self._transports: Dict[str, asyncio.Future] = {}
        self._stopping = False
        self._stack = contextlib.AsyncExitStack() 

def _update_sidecar(sidecar_fname, key, val):
    """Update a sidecar JSON file with a given key/value pair.

    Parameters
    ----------
    sidecar_fname : str | os.PathLike
        Full name of the data file
    key : str
        The key in the sidecar JSON file. E.g. "PowerLineFrequency"
    val : str
        The corresponding value to change to in the sidecar JSON file.
    """
    with open(sidecar_fname, "r") as fin:
        sidecar_json = json.load(fin)
    sidecar_json[key] = val
    with open(sidecar_fname, "w") as fout:
        json.dump(sidecar_json, fout) 

def read(self, filenames, encoding=None):
        """Read and parse a filename or an iterable of filenames.

        Files that cannot be opened are silently ignored; this is
        designed so that you can specify an iterable of potential
        configuration file locations (e.g. current directory, user's
        home directory, systemwide directory), and all existing
        configuration files in the iterable will be read.  A single
        filename may also be given.

        Return list of successfully read files.
        """
        if isinstance(filenames, (str, bytes, os.PathLike)):
            filenames = [filenames]
        read_ok = []
        for filename in filenames:
            try:
                with open(filename, encoding=encoding) as fp:
                    self._read(fp, filename)
            except OSError:
                continue
            if isinstance(filename, os.PathLike):
                filename = os.fspath(filename)
            read_ok.append(filename)
        return read_ok 

def clean_files(files):
	"""Generates tuples with a ``file``-like object and a close indicator.

	This is a generator of tuples, where the first element is the file object
	and the second element is a boolean which is True if this module opened the
	file (and thus should close it).

	Raises
	------
	OSError : Accessing the given file path failed

	Parameters
	----------
	files : Union[str, bytes, os.PathLike, io.IOBase, int, collections.abc.Iterable]
		Collection or single instance of a filepath and file-like object
	"""
	if not isinstance(files, path_types) and not hasattr(files, "read"):
		for f in files:
			yield clean_file(f)
	else:
		yield clean_file(files) 

def updateFileObservers(self):
        obs = getFileObserver()
        newObservedPaths = defaultdict(list)
        for fontItemInfo in self.project.fonts:
            fontPath = fontItemInfo.fontKey[0]
            if not fontPath.exists():
                # We can't observe a non-existing path. This can happen
                # if the project file contains a wrong source path.
                # We don't want to stop loading other fonts that do have a
                # correct path so we won't complain here.
                continue
            newObservedPaths[fontPath].append(fontItemInfo)
            if fontItemInfo.font is not None:
                for path in fontItemInfo.font.getExternalFiles():
                    assert isinstance(path, os.PathLike)
                    newObservedPaths[path].append(fontItemInfo)
        newPaths = set(newObservedPaths)
        oldPaths = set(self.observedPaths)
        for path in newPaths - oldPaths:
            obs.addObserver(path, self._fileChanged)
        for path in oldPaths - newPaths:
            obs.removeObserver(path, self._fileChanged)
        self.observedPaths = newObservedPaths 

def get_hash(file, key=None):
    if file is None:
        raise Exception(f"Can't get hash of file {file}: file is 'None'")

    if isinstance(file, (str, bytes, os.PathLike)):
        if isfile(file):
            with open(file, 'rb') as f:
                data = f.read()
        elif isdir(file):
            return get_dir_hash(file)
        else:
            return ''
    else:
        openedfile = file.open()
        data = openedfile.read()
        openedfile.seek(0)

    return compute_hash(data, key) 

def setUp(self):
        super().setUp()

        # Mock os.path.basename() because since Python 3.6, it fails with an
        # error when a mock is passed to it ("TypeError: expected str, bytes or
        # os.PathLike object, not MagicMock").
        self.os_path_basename_mock = mock.Mock()
        self.patch(
            'retdec.tools.decompiler.os.path.basename',
            self.os_path_basename_mock
        )

        # Mock Decompiler so that when it is instantiated, it returns our
        # decompiler that can be used in the tests.
        self.decompiler = mock.MagicMock(spec_set=Decompiler)
        self.DecompilerMock = mock.Mock()
        self.DecompilerMock.return_value = self.decompiler
        self.patch(
            'retdec.tools.decompiler.Decompiler',
            self.DecompilerMock
        ) 

def __init__(
        self,
        path: typing.Union[bytes, str, os.PathLike],
        pre_load: bool = None,
        fps: int = None,
        *_,
        **__,
    ):
        assert os.path.isfile(path), f"video [{path}] not existed"
        self.path: str = str(path)
        self.data: typing.Optional[typing.Tuple[VideoFrame]] = tuple()

        self.fps: int = fps
        if fps:
            video_path = os.path.join(tempfile.mkdtemp(), f"tmp_{fps}.mp4")
            logger.debug(f"convert video, and bind path to {video_path}")
            toolbox.fps_convert(fps, self.path, video_path, constants.FFMPEG)
            self.path = video_path

        with toolbox.video_capture(self.path) as cap:
            self.frame_count = toolbox.get_frame_count(cap)
            self.frame_size = toolbox.get_frame_size(cap)

        if pre_load:
            self.load_frames() 

def validate_urls(wrapped, instance, args, kwargs):
    """Enforces argument named "urls" to be a list of relative paths."""
    try:
        # Use -1 since self is not included in the args.
        urls = args[getfullargspec(wrapped).args.index("urls") - 1]
    except IndexError:
        urls = kwargs.get("urls")
    # Check that URLS is really a list of strings.
    if not isinstance(urls, list):
        raise TypeError("Argument urls must be a list of strings or path-like objects")
    if not all(isinstance(url, (str, PathLike)) for url in urls):
        raise TypeError("Argument urls must be a list of strings or path-like objects")
    # Check that all URLS are relative.
    if any(PurePath(url).is_absolute() for url in urls):
        raise ValueError("Paths must be relative.")
    return wrapped(*args, *kwargs) 

def __init__(
            self,
            pt_exec: Union[List[str], List[bytes]],
            state: Union[None, str, bytes, os.PathLike],
            *,
            exit_on_stdin_close: bool = True,
    ) -> None:
        super().__init__(
            pt_exec, state, exit_on_stdin_close=exit_on_stdin_close)
        self._transport_opts: Dict[str, ServerTransportOptions] = {} 

def __init__(
            self,
            pt_exec: Union[List[str], List[bytes]],
            state: Union[None, str, bytes, os.PathLike],
            forward_host: Union[str,
                                ipaddress.IPv4Address,
                                ipaddress.IPv6Address],
            forward_port: int,
            *,
            exit_on_stdin_close: bool = True,
    ) -> None:
        """Create the adapter.

        Args:
            pt_exec: The pluggable transport command line to execute. This has
                to be a list of str / bytes, since
                :func:`asyncio.create_subprocess_exec` does not accept an
                entire command line as a string. On non-Windows platforms
                :func:`shlex.split` can be used to split a command line string
                into a list, while on Windows it's a bit more complicated.
            state: The state directory. This is a directory where the PT is
                allowed to store state. Either specify a path (which
                is not required to exist, in which case the PT will create
                the directory), or specify ``None`` to use a temporary
                directory created using :mod:`tempfile`. For servers, using
                an actual persistent location is recommended.
            forward_host: IP address or host name to forward unobfuscated
                traffic to.
            forward_port: Port number to forward unobfuscated traffic to.
            exit_on_stdin_close: Whether closing the PT's STDIN indicates the
                PT should gracefully exit.
        """
        super().__init__(
            pt_exec, state, exit_on_stdin_close=exit_on_stdin_close)
        self._orport_host = forward_host
        self._orport_port = forward_port 

def load_metadata(source_dir: Union[os.PathLike, str]) -> dict:
    """
    Load the given metadata.json which was saved during the ``serializer.dump``
    will return the loaded metadata as a dict, or empty dict if no file was found

    Parameters
    ----------
    source_dir: Union[os.PathLike, str]
        Directory of the saved model, As with serializer.load(source_dir) this
        source_dir can be the top level, or the first dir into the serialized model.

    Returns
    -------
    dict

    Raises
    ------
    FileNotFoundError
        If a 'metadata.json' file isn't found in or above the supplied ``source_dir``
    """
    # Since this function can take the top level dir, or a dir directly
    # into the first step of the pipeline, we need to check both for metadata
    possible_paths = [
        os.path.join(source_dir, "metadata.json"),
        os.path.join(source_dir, "..", "metadata.json"),
    ]
    path = next((path for path in possible_paths if os.path.exists(path)), None)

    if path:
        with open(path, "r") as f:
            return simplejson.load(f)
    else:
        raise FileNotFoundError(
            f"Metadata file in source dir: '{source_dir}' not found in or up one directory."
        ) 

def __init__(
            self,
            pt_exec: Union[List[str], List[bytes]],
            state: Union[None, str, bytes, os.PathLike],
            transports: List[str],
            proxy: str = None,
            *,
            exit_on_stdin_close: bool = True,
    ) -> None:
        """Create the adapter.

        Args:
            pt_exec: The pluggable transport command line to execute. This has
                to be a list of str / bytes, since
                :func:`asyncio.create_subprocess_exec` does not accept an
                entire command line as a string. On non-Windows platforms
                :func:`shlex.split` can be used to split a command line string
                into a list, while on Windows it's a bit more complicated.
            state: The state directory. This is a directory where the PT is
                allowed to store state. Either specify an absolute path (which
                is not required to exist, in which case the PT will create
                the directory), or specify ``None`` to use a temporary
                directory created using :mod:`tempfile`.
            transports: a list of client transports the PT should initialize.
                PTs will ignore names they don't recognize.
            proxy: The upstream proxy to use. Must be specified in the URI
                format:
                ``<proxy_type>://[<user_name>[:<password>][@]<ip>:<port>``.
            exit_on_stdin_close: Whether closing the PT's STDIN indicates the
                PT should gracefully exit.
        """
        super().__init__(
            pt_exec, state, exit_on_stdin_close=exit_on_stdin_close)
        for transport in transports:
            str_utils.validate_transport_name(transport)
            self._transports[transport] = asyncio.Future()
        self._proxy = proxy 

def stream_files(files, *, chunk_size=default_chunk_size):
	"""Gets a buffered generator for streaming files.

	Returns a buffered generator which encodes a file or list of files as
	:mimetype:`multipart/form-data` with the corresponding headers.

	Parameters
	----------
	files : Union[str, bytes, os.PathLike, io.IOBase, int, collections.abc.Iterable]
		The file(s) to stream
	chunk_size : int
		Maximum size of each stream chunk
	"""
	stream = FilesStream(files, chunk_size=chunk_size)
	return stream.body(), stream.headers() 

def dump(obj: object, dest_dir: Union[os.PathLike, str], metadata: dict = None):
    """
    Serialize an object into a directory, the object must be pickle-able.

    Parameters
    ----------
    obj
        The object to dump. Must be pickle-able.
    dest_dir: Union[os.PathLike, str]
        The directory to which to save the model metadata: dict - any additional
        metadata to be saved alongside this model if it exists, will be returned
        from the corresponding "load" function
    metadata: Optional dict of metadata which will be serialized to a file together
        with the model, and loaded again by :func:`load_metadata`.

    Returns
    -------
    None

    Example
    -------

    >>> from sklearn.pipeline import Pipeline
    >>> from sklearn.decomposition import PCA
    >>> from gordo.machine.model.models import KerasAutoEncoder
    >>> from gordo import serializer
    >>> from tempfile import TemporaryDirectory
    >>> pipe = Pipeline([
    ...     ('pca', PCA(3)),
    ...     ('model', KerasAutoEncoder(kind='feedforward_hourglass'))])
    >>> with TemporaryDirectory() as tmp:
    ...     serializer.dump(obj=pipe, dest_dir=tmp)
    ...     pipe_clone = serializer.load(source_dir=tmp)
    """
    with open(os.path.join(dest_dir, "model.pkl"), "wb") as m:
        pickle.dump(obj, m)
    if metadata is not None:
        with open(os.path.join(dest_dir, "metadata.json"), "w") as f:
            simplejson.dump(metadata, f, default=str) 

def output_dir(self) -> Union[str, os.PathLike]:
        return (
            Path.home()
            / "zookeeper-logs"
            / self.dataset.__class__.__name__
            / self.__class__.__name__
            / datetime.now().strftime("%Y%m%d_%H%M")
        ) 

def check_cache(self, model_register_dir: Union[os.PathLike, str]):
        """
        Checks if the model is cached, and returns its path if it exists.

        Parameters
        ----------
        model_register_dir: [os.PathLike, None]
            The register dir where the model lies.
        cache_key: str
            A 512 byte hex value as a string based on the content of the parameters.

         Returns
        -------
        Union[os.PathLike, None]:
            The path to the cached model, or None if it does not exist.
        """
        existing_model_location = disk_registry.get_value(
            model_register_dir, self.cache_key
        )

        # Check that the model is actually there
        if existing_model_location and Path(existing_model_location).exists():
            logger.debug(
                f"Found existing model at path {existing_model_location}, returning it"
            )
            return existing_model_location
        elif existing_model_location:
            logger.warning(
                f"Found that the model-path {existing_model_location} stored in the "
                f"registry did not exist."
            )
            return None
        else:
            logger.info(
                f"Did not find the model with key {self.cache_key} in the register at "
                f"{model_register_dir}."
            )
            return None 

def restore(
        self,
        conn: edgedb.BlockingIOConnection,
        dumpfn: os.PathLike
    ) -> None:
        with open(dumpfn, 'rb') as f:
            header, reader = self._parse(f)

            conn._restore(
                header=header,
                data_gen=reader,
            ) 

def _get_adb_path(cls) -> os.PathLike:
        """Return adb path based on operating system detected during import"""

        if runtime_platform == PLATFORM.OSX:
            path = os.path.join(pkg_root, "bin", "osx", "adb")
        elif runtime_platform == PLATFORM.WIN:
            path = os.path.join(pkg_root, "bin", "win", "adb.exe")
        elif runtime_platform == PLATFORM.LINUX:
            path = os.path.join(pkg_root, "bin", "linux", "adb")
        else:
            raise RuntimeError("Unsupported runtime platform")

        return pathlib.Path(path) 

def os_fspath(path):
        """Return the path representation of a path-like object.
        If str or bytes is passed in, it is returned unchanged. Otherwise the
        os.PathLike interface is used to get the path representation. If the
        path representation is not str or bytes, TypeError is raised. If the
        provided path is not str, bytes, or os.PathLike, TypeError is raised.
        """
        if isinstance(path, (unicode, bytes)):
            return path

        # Work from the object's type to match method resolution of other magic
        # methods.
        path_type = type(path)
        try:
            path_repr = path_type.__fspath__(path)
        except AttributeError:
            if hasattr(path_type, '__fspath__'):
                raise
            elif PurePath is not None and issubclass(path_type, PurePath):
                return _PurePath__fspath__(path)
            else:
                raise TypeError("expected str, bytes or os.PathLike object, "
                                "not " + path_type.__name__)
        if isinstance(path_repr, (unicode, bytes)):
            return path_repr
        else:
            raise TypeError("expected {}.__fspath__() to return str or bytes, "
                            "not {}".format(path_type.__name__,
                                            type(path_repr).__name__))