Python pathlib2.Path() Examples

def _form_conf_path(self, conf_var, prefix):
        """Form configuration file path with configdir if applicable
        :param CliContext self: this
        :param any conf_var: conf var
        :param str prefix: configuration file prefix
        :rtype: pathlib.Path
        :return: new configuration file path
        """
        # use configdir if available
        if conf_var is None:
            cd = self.configdir or '.'
            pathyaml = pathlib.Path(cd, '{}.yaml'.format(prefix))
            if pathyaml.exists():
                return pathyaml
            path = pathlib.Path(cd, '{}.yml'.format(prefix))
            if path.exists():
                return path
            path = pathlib.Path(cd, '{}.json'.format(prefix))
            if path.exists():
                return path
            return pathyaml
        else:
            return conf_var 

def test_no_setup_py_exits_but_pyproject_toml_does(cmd, initproj):
    initproj(
        "pkg123-0.7",
        filedefs={
            "tox.ini": """
            [testenv]
            commands=python -c "2 + 2"
        """,
        },
    )
    os.remove("setup.py")
    pathlib2.Path("pyproject.toml").touch()
    result = cmd()
    result.assert_fail()
    assert any(
        re.match(r".*ERROR.*pyproject.toml file found.*", line) for line in result.outlines
    ), result.outlines
    assert any(
        re.match(r".*To use a PEP 517 build-backend you are required to*", line)
        for line in result.outlines
    ), result.outlines 

def wait_for_env_startup(process):
    """the environments will write files once they are up"""
    signal_files = [Path() / "a", Path() / "b"]
    found = False
    while True:
        if process.poll() is not None:
            break
        for signal_file in signal_files:
            if not signal_file.exists():
                break
        else:
            found = True
            break
    if not found or process.poll() is not None:
        missing = [f for f in signal_files if not f.exists()]
        out, _ = process.communicate()
        assert len(missing), out
        assert False, out 

def run(cmd, package):
    result = cmd("--sdistonly", "-e", "py", "-v", "-v")
    result.assert_success(is_run_test_env=False)
    package_venv = (Path() / ".tox" / ".package").resolve()
    assert ".package create: {}".format(package_venv) in result.outlines, result.out
    assert "write config to {}".format(package_venv / ".tox-config1") in result.out, result.out
    package_path = (Path() / ".tox" / "dist" / package).resolve()
    assert package_path.exists()

    package_path.unlink()

    # second call re-uses
    result2 = cmd("--sdistonly", "-e", "py", "-v", "-v")

    result2.assert_success(is_run_test_env=False)
    assert (
        ".package reusing: {}".format(package_venv) in result2.outlines
    ), "Second call output:\n{}First call output:\n{}".format(result2.out, result.out)
    assert package_path.exists() 

def make_numbered_dir(root: Path, prefix: str) -> Path:
    """create a directory with an increased number as suffix for the given prefix"""
    for i in range(10):
        # try up to 10 times to create the folder
        max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)
        new_number = max_existing + 1
        new_path = root.joinpath("{}{}".format(prefix, new_number))
        try:
            new_path.mkdir()
        except Exception:
            pass
        else:
            _force_symlink(root, prefix + "current", new_path)
            return new_path
    else:
        raise OSError(
            "could not create numbered dir with prefix "
            "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root)
        ) 

def _force_symlink(
    root: Path, target: Union[str, PurePath], link_to: Union[str, Path]
) -> None:
    """helper to create the current symlink

    it's full of race conditions that are reasonably ok to ignore
    for the context of best effort linking to the latest test run

    the presumption being that in case of much parallelism
    the inaccuracy is going to be acceptable
    """
    current_symlink = root.joinpath(target)
    try:
        current_symlink.unlink()
    except OSError:
        pass
    try:
        current_symlink.symlink_to(link_to)
    except Exception:
        pass 

def ensure_extended_length_path(path: Path) -> Path:
    """Get the extended-length version of a path (Windows).

    On Windows, by default, the maximum length of a path (MAX_PATH) is 260
    characters, and operations on paths longer than that fail. But it is possible
    to overcome this by converting the path to "extended-length" form before
    performing the operation:
    https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation

    On Windows, this function returns the extended-length absolute version of path.
    On other platforms it returns path unchanged.
    """
    if sys.platform.startswith("win32"):
        path = path.resolve()
        path = Path(get_extended_length_path_str(str(path)))
    return path 

def test_islink(self):
        if os.name == 'nt':
            raise unittest.SkipTest('symlink')
        assert_islink(self.link_path, to=self.file_path)
        assert_islink(pathlib.Path(self.link_path),
                      to=pathlib.Path(self.file_path))
        assert_not_islink(self.file_path)
        
        with self.assertRaises(AssertionError) as c:
            assert_islink(self.file_path)
        self.assertIn('not a symlink', str(c.exception))
        
        with self.assertRaises(AssertionError) as c:
            assert_islink(self.link_path, to=self.dir_path)
        self.assertIn('target of', str(c.exception))
        
        with self.assertRaises(AssertionError):
            assert_not_islink(self.link_path) 

def assert_islink(path, to=None, msg=None):
    """Assert that path exists and is a symlink.
    
    If to is specified, also check that it is the target of the symlink.
    """
    path = _strpath(path)
    st = _stat_for_assert(path, False, msg)
    if not stat.S_ISLNK(st.st_mode):
        if msg is None:
            msg = "Path exists, but is not a symlink: %r" % path
        raise AssertionError(msg)
    
    if to is not None:
        to = _strpath(to)
        target = os.readlink(path)
        # TODO: Normalise the target to an absolute path?
        if target != to:
            if msg is None:
                msg = _link_target_msg.format(path=path, expected=to, actual=target)
            raise AssertionError(msg) 

def get_custom_dict():
        folder = Path(".").cwd().parts[-1]
        hdf_file = Path(".").cwd().parents[1] / folder
        hdf_file = str(hdf_file) + ".h5"
        if Path(hdf_file).exists():
            hdf = FileHDFio(hdf_file)
            custom_dict = GenericParameters()
            for k, v in zip(
                hdf[folder + "/input/custom_dict/data_dict"]["Parameter"],
                hdf[folder + "/input/custom_dict/data_dict"]["Value"],
            ):
                custom_dict[k] = v
            return custom_dict
        elif Path("input.json").exists():
            with open("input.json") as f:
                return json.load(f)
        else:
            print(hdf_file, "not found")
            return None 

def dump_flight_to_csv(flight, track_filename_local, thermals_filename_local):
    """Dumps flight data to CSV files.

    Args:
        flight: an igc_lib.Flight, the flight to be written
        track_filename_local: a string, the name of the output CSV with track data
        thermals_filename_local: a string, the name of the output CSV with thermal data
    """
    track_filename = Path(track_filename_local).expanduser().absolute()
    with track_filename.open('wt') as csv:
        csv.write(u"timestamp,lat,lon,bearing,bearing_change_rate,"
                  u"gsp,flying,circling\n")
        for fix in flight.fixes:
            csv.write(u"%f,%f,%f,%f,%f,%f,%s,%s\n" % (
                fix.timestamp, fix.lat, fix.lon,
                fix.bearing, fix.bearing_change_rate,
                fix.gsp, str(fix.flying), str(fix.circling)))

    thermals_filename = Path(thermals_filename_local).expanduser().absolute()
    with thermals_filename.open('wt') as csv:
        csv.write(u"timestamp_enter,timestamp_exit\n")
        for thermal in flight.thermals:
            csv.write(u"%f,%f\n" % (
                thermal.enter_fix.timestamp, thermal.exit_fix.timestamp)) 

def maybe_delete_a_numbered_dir(path: Path) -> None:
    """removes a numbered directory if its lock can be obtained and it does not seem to be in use"""
    path = ensure_extended_length_path(path)
    lock_path = None
    try:
        lock_path = create_cleanup_lock(path)
        parent = path.parent

        garbage = parent.joinpath("garbage-{}".format(uuid.uuid4()))
        path.rename(garbage)
        rm_rf(garbage)
    except OSError:
        #  known races:
        #  * other process did a cleanup at the same time
        #  * deletable folder was found
        #  * process cwd (Windows)
        return
    finally:
        # if we created the lock, ensure we remove it even if we failed
        # to properly remove the numbered dir
        if lock_path is not None:
            try:
                lock_path.unlink()
            except OSError:
                pass 

def get_oasislmf_config_path(model_id=None):
    conf_var = settings.get('worker', 'oasislmf_config', fallback=None)
    if not model_id:
        model_id = settings.get('worker', 'model_id', fallback=None)

    if conf_var:
        return conf_var

    if model_id:
        model_root = settings.get('worker', 'model_data_directory', fallback='/var/oasis/')
        model_specific_conf = Path(model_root, '{}-oasislmf.json'.format(model_id))
        if model_specific_conf.exists():
            return str(model_specific_conf)

    return str(Path(model_root, 'oasislmf.json'))


# Send notification back to the API Once task is read from Queue 

def compute_md5_for_file(file, as_base64, blocksize=65536):
    # type: (pathlib.Path, bool, int) -> str
    """Compute MD5 hash for file
    :param pathlib.Path file: file to compute md5 for
    :param bool as_base64: return as base64 encoded string
    :param int blocksize: block size in bytes
    :rtype: str
    :return: md5 for file
    """
    hasher = hashlib.md5()
    if isinstance(file, pathlib.Path):
        file = str(file)
    with open(file, 'rb') as filedesc:
        while True:
            buf = filedesc.read(blocksize)
            if not buf:
                break
            hasher.update(buf)
        if as_base64:
            return base64_encode_string(hasher.digest())
        else:
            return hasher.hexdigest() 

def compute_sha256_for_file(file, as_base64, blocksize=65536):
    # type: (pathlib.Path, bool, int) -> str
    """Compute SHA256 hash for file
    :param pathlib.Path file: file to compute md5 for
    :param bool as_base64: return as base64 encoded string
    :param int blocksize: block size in bytes
    :rtype: str
    :return: SHA256 for file
    """
    hasher = hashlib.sha256()
    if isinstance(file, pathlib.Path):
        file = str(file)
    with open(file, 'rb') as filedesc:
        while True:
            buf = filedesc.read(blocksize)
            if not buf:
                break
            hasher.update(buf)
        if as_base64:
            return base64_encode_string(hasher.digest())
        else:
            return hasher.hexdigest() 

def run(model_path, model_name, tenant_id, service_principal_id,
        service_principal_password, subscription_id, resource_group, workspace, tags):
    auth_args = {
        'tenant_id': tenant_id,
        'service_principal_id': service_principal_id,
        'service_principal_password': service_principal_password
    }

    ws_args = {
        'auth': ServicePrincipalAuthentication(**auth_args),
        'subscription_id': subscription_id,
        'resource_group': resource_group
    }

    ws = Workspace.get(workspace, **ws_args)

    print(ws.get_details())

    print('\nSaving model {} to {}'.format(model_path, model_name))

    # Model Path needs to be relative
    model_path = relpath(model_path, '.')

    model = Model.register(ws, model_name=model_name, model_path=model_path, tags=tags)
    print('Done!') 

def create_from_file(filename, config_class=FlightParsingConfig):
        """Creates an instance of Flight from a given file.

        Args:
            filename: a string, the name of the input IGC file
            config_class: a class that implements FlightParsingConfig

        Returns:
            An instance of Flight built from the supplied IGC file.
        """
        config = config_class()
        fixes = []
        a_records = []
        i_records = []
        h_records = []
        abs_filename = Path(filename).expanduser().absolute()
        with abs_filename.open('r', encoding="ISO-8859-1") as flight_file:
            for line in flight_file:
                line = line.replace('\n', '').replace('\r', '')
                if not line:
                    continue
                if line[0] == 'A':
                    a_records.append(line)
                elif line[0] == 'B':
                    fix = GNSSFix.build_from_B_record(line, index=len(fixes))
                    if fix is not None:
                        if fixes and math.fabs(fix.rawtime - fixes[-1].rawtime) < 1e-5:
                            # The time did not change since the previous fix.
                            # Ignore this fix.
                            pass
                        else:
                            fixes.append(fix)
                elif line[0] == 'I':
                    i_records.append(line)
                elif line[0] == 'H':
                    h_records.append(line)
                else:
                    # Do not parse any other types of IGC records
                    pass
        flight = Flight(fixes, a_records, h_records, i_records, config)
        return flight 

def dump_flight_to_kml(flight, kml_filename_local):
    """Dumps the flight to KML format.

    Args:
        flight: an igc_lib.Flight, the flight to be saved
        kml_filename_local: a string, the name of the output file
    """
    assert flight.valid
    kml = simplekml.Kml()

    def add_point(name, fix):
        kml.newpoint(name=name, coords=[(fix.lon, fix.lat)])

    coords = []
    for fix in flight.fixes:
        coords.append((fix.lon, fix.lat))
    kml.newlinestring(coords=coords)

    add_point(name="Takeoff", fix=flight.takeoff_fix)
    add_point(name="Landing", fix=flight.landing_fix)

    for i, thermal in enumerate(flight.thermals):
        add_point(name="thermal_%02d" % i, fix=thermal.enter_fix)
        add_point(name="thermal_%02d_END" % i, fix=thermal.exit_fix)
        kml_filename = Path(kml_filename_local).expanduser().absolute()
    kml.save(kml_filename.as_posix()) 

def dump_thermals_to_cup_file(flight, cup_filename_local):
    """Dump flight's thermals to a .cup file (SeeYou).

    Args:
        flight: an igc_lib.Flight, the flight to be written
        cup_filename_local: a string, the name of the file to be written.
    """
    cup_filename = Path(cup_filename_local).expanduser().absolute()
    with cup_filename.open('wt') as wpt:
        wpt.write(u'name,code,country,lat,')
        wpt.write(u'lon,elev,style,rwdir,rwlen,freq,desc,userdata,pics\n')

        def write_fix(name, fix):
            lat = _degrees_float_to_degrees_minutes_seconds(fix.lat, 'lat')
            lon = _degrees_float_to_degrees_minutes_seconds(fix.lon, 'lon')
            wpt.write(u'"%s",,,%02d%02d.%03d%s,' % (
                name, lat.degrees, lat.minutes,
                int(round(lat.seconds/60.0*1000.0)), lat.hemisphere))
            wpt.write(u'%03d%02d.%03d%s,%fm,,,,,,,' % (
                lon.degrees, lon.minutes,
                int(round(lon.seconds/60.0*1000.0)), lon.hemisphere,
                fix.gnss_alt))
            wpt.write(u'\n')

        for i, thermal in enumerate(flight.thermals):
            write_fix(u'%02d' % i, thermal.enter_fix)
            write_fix(u'%02d_END' % i, thermal.exit_fix) 

def setup_plugin(name):
    target = os.path.expandvars(os.path.expanduser("~/.{0}".format(name)))
    this = Path(__file__).absolute().parent
    plugin_dir = this / "test_artifacts" / name
    plugin_uri = plugin_dir.as_uri()
    if not "file:///" in plugin_uri and "file:/" in plugin_uri:
        plugin_uri = plugin_uri.replace("file:/", "file:///")
    out = subprocess.check_output(
        ["git", "clone", plugin_uri, Path(target).as_posix()]
    )
    print(out, file=sys.stderr) 

def tracked_tmpdir_factory():
    def create_tmpdir():
        return Path(_create_tracked_dir())

    yield create_tmpdir 

def tracked_tmpdir():
    temp_path = _create_tracked_dir()
    yield Path(temp_path) 

def pathlib_tmpdir(request, tmpdir):
    yield Path(str(tmpdir))
    try:
        tmpdir.remove(ignore_errors=True)
    except Exception:
        pass 

def __init__(self, datadir, original_datadir, request):
        """
        :type datadir: Path
        :type original_datadir: Path
        :type request: FixtureRequest
        """
        self.request = request
        self.datadir = datadir
        self.original_datadir = original_datadir 

def datadir(original_datadir, tmpdir):
    # Method from: https://github.com/gabrielcnr/pytest-datadir
    # License: MIT
    import shutil
    result = Path(str(tmpdir.join(original_datadir.stem)))
    if original_datadir.is_dir():
        shutil.copytree(str(original_datadir), str(result))
    else:
        result.mkdir()
    return result 

def original_datadir(request):
    # Method from: https://github.com/gabrielcnr/pytest-datadir
    # License: MIT
    import os.path
    return Path(os.path.splitext(request.module.__file__)[0]) 

def _rsa_decrypt_string_with_pfx(ciphertext, config):
    # type: (str, dict) -> str
    """RSA decrypt a string
    :param str ciphertext: cipher text in base64
    :param dict config: configuration dict
    :rtype: str
    :return: decrypted cipher text
    """
    if util.is_none_or_empty(ciphertext):
        raise ValueError('invalid ciphertext to decrypt')
    pfxfile = settings.batch_shipyard_encryption_pfx_filename(config)
    pfx_passphrase = settings.batch_shipyard_encryption_pfx_passphrase(config)
    pemfile = derive_private_key_pem_from_pfx(pfxfile, pfx_passphrase, None)
    if pemfile is None:
        raise RuntimeError('cannot decrypt without valid private key')
    cleartext = None
    try:
        data = util.base64_decode_string(ciphertext)
        proc = subprocess.Popen(
            ['openssl', 'rsautl', '-decrypt', '-inkey', pemfile],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        cleartext = proc.communicate(input=data)[0]
    finally:
        fp = pathlib.Path(pemfile)
        if fp.exists():
            fp.unlink()
    return cleartext 

def _read_config_file(self, config_file):
        # type: (CliContext, pathlib.Path) -> None
        """Read a yaml/json file into self.config
        :param CliContext self: this
        :param pathlib.Path config_file: config file to load
        """
        with config_file.open('r') as f:
            if self.config is None:
                self.config = ruamel.yaml.load(
                    f, Loader=ruamel.yaml.RoundTripLoader)
            else:
                self.config = convoy.util.merge_dict(
                    self.config,
                    ruamel.yaml.load(f, Loader=ruamel.yaml.RoundTripLoader)) 

def ensure_pathlib_conf(conf):
        # type: (Any) -> pathlib.Path
        """Ensure conf object is a pathlib object
        :param str or pathlib.Path or None conf: conf object
        :rtype: pathlib.Path or None
        :return: conf object as pathlib
        """
        if conf is not None and not isinstance(conf, pathlib.Path):
            conf = pathlib.Path(conf)
        return conf 

def load_dataset(base_path, dataset, split=[8, 1, 1]):
    # normalize splits
    splits = np.array(split) / np.sum(np.array(split))

    # find labels - parent folder names
    labels = {}
    for (_, dirs, _) in os.walk(base_path):
        print('found {}'.format(dirs))
        labels = { k: v for (v, k) in enumerate(dirs) }
        print('using {}'.format(labels))
        break
        
    # load all files along with idx label
    print('loading dataset from {}'.format(dataset))
    with open(dataset, 'r') as d:
        data = [(str(Path(f.strip()).absolute()), labels[Path(f.strip()).parent.name]) for f in d.readlines()]

    print('dataset size: {}\nsuffling data...'.format(len(data)))
    
    # shuffle data
    shuffle(data)
    
    print('splitting data...')
    # split data
    train_idx = int(len(data) * splits[0])
    eval_idx = int(len(data) * splits[1])
    
    return data[:train_idx], \
            data[train_idx:train_idx + eval_idx], \
            data[train_idx + eval_idx:], \
            labels

#@print_info