Python os.path.sep() Examples

The following are code examples for showing how to use os.path.sep(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: spleeter   Author: deezer   File: __init__.py    MIT License 6 votes vote down vote up
def get(self, model_directory):
        """ Ensures required model is available at given location.

        :param model_directory: Expected model_directory to be available.
        :raise IOError: If model can not be retrieved.
        """
        # Expend model directory if needed.
        if not isabs(model_directory):
            model_directory = join(self.DEFAULT_MODEL_PATH, model_directory)
        # Download it if not exists.
        model_probe = join(model_directory, self.MODEL_PROBE_PATH)
        if not exists(model_probe):
            if not exists(model_directory):
                makedirs(model_directory)
                self.download(
                    model_directory.split(sep)[-1],
                    model_directory)
                self.writeProbe(model_directory)
        return model_directory 
Example 2
Project: imgcomp-cvpr   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 6 votes vote down vote up
def create_unique_log_dir(config_rel_paths, log_dir_root, line_breaking_chars_pat=r'[-]', restore_dir=None):
    """
    0117_1704 [email protected]_med_8e*5_deePer_b50_noHM_C16 [email protected]_res_shallow [email protected]@[email protected]@0115_1340
    :param config_rel_paths:
    :param log_dir_root:
    :param line_breaking_chars_pat:
    :return:
    """
    if any(':' in config_rel_path for config_rel_path in config_rel_paths):
        raise ValueError('":" not allowed in paths, got {}'.format(config_rel_paths))

    def prep_path(p):
        p = p.replace(path.sep, '@')
        return re.sub(line_breaking_chars_pat, '*', p)

    postfix_dir_name = ' '.join(map(prep_path, config_rel_paths))
    if restore_dir:
        restore_dir_root, restore_job_component = _split_log_dir(restore_dir)
        restore_dir_root = restore_dir_root.replace(path.sep, '@')
        restore_job_id = log_date_from_log_dir(restore_job_component)
        postfix_dir_name += ' {restore_prefix}{root}@{job_id}'.format(
                restore_prefix=_RESTORE_PREFIX, root=restore_dir_root, job_id=restore_job_id)
    return _mkdir_threadsafe_unique(log_dir_root, datetime.now(), postfix_dir_name) 
Example 3
Project: imgcomp-cvpr   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 6 votes vote down vote up
def _split_log_dir(log_dir):
    """
    given
        some/path/to/job/dir/0101_1818 ae_config pc_config/ckpts
    or
        some/path/to/job/dir/0101_1818 ae_config pc_config
    returns
        tuple some/path/to/job/dir, 0101_1818 ae_config pc_config
    """
    log_dir_root = []
    job_component = None

    for comp in log_dir.split(path.sep):
        try:
            log_date_from_log_dir(comp)
            job_component = comp
            break  # this component is an actual log dir. stop and return components
        except ValueError:
            log_dir_root.append(comp)

    assert job_component is not None, 'Invalid log_dir: {}'.format(log_dir)
    return path.sep.join(log_dir_root), job_component 
Example 4
Project: imgcomp-cvpr   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 6 votes vote down vote up
def config_paths_from_log_dir(log_dir, base_dirs):
    log_dir = path.basename(log_dir.strip(path.sep))

    # log_dir == {now} {netconfig} {probconfig} [[email protected][email protected]_YYYY], get [netconfig, probconfig]
    comps = log_dir.split(' ')
    assert is_log_date(comps[0]), 'Invalid log_dir: {}'.format(log_dir)
    comps = [c for c in comps[1:] if _RESTORE_PREFIX not in c]
    assert len(comps) <= len(base_dirs), 'Expected as many config components as base dirs: {}, {}'.format(
            comps, base_dirs)

    def get_real_path(base, prepped_p):
        p_glob = prepped_p.replace('@', path.sep)
        p_glob = path.join(base, p_glob)  # e.g., ae_configs/p_glob
        glob_matches = glob.glob(p_glob)
        # We always only replace one character with *, so filter for those.
        # I.e. lr1e-5 will become lr1e*5, which will match lr1e-5 but also lr1e-4.5
        glob_matches_of_same_len = [g for g in glob_matches if len(g) == len(p_glob)]
        if len(glob_matches_of_same_len) != 1:
            raise ValueError('Cannot find config on disk: {} (matches: {})'.format(p_glob, glob_matches_of_same_len))
        return glob_matches_of_same_len[0]

    return tuple(get_real_path(base_dir, comp) for base_dir, comp in zip(base_dirs, comps)) 
Example 5
Project: L3C-PyTorch   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 6 votes vote down vote up
def _split_log_dir(log_dir):
    """
    given
        some/path/to/job/dir/0101_1818 ae_config pc_config/ckpts
    or
        some/path/to/job/dir/0101_1818 ae_config pc_config
    returns
        tuple some/path/to/job/dir, 0101_1818 ae_config pc_config
    """
    log_dir_root = []
    job_component = None

    for comp in log_dir.split(path.sep):
        try:
            log_date_from_log_dir(comp)
            job_component = comp
            break  # this component is an actual log dir. stop and return components
        except ValueError:
            log_dir_root.append(comp)

    assert job_component is not None, 'Invalid log_dir: {}'.format(log_dir)
    return path.sep.join(log_dir_root), job_component 
Example 6
Project: Repobot   Author: Desgard   File: fix_absolute_import.py    MIT License 6 votes vote down vote up
def probably_a_local_import(self, imp_name):
        """
        Like the corresponding method in the base class, but this also
        supports Cython modules.
        """
        if imp_name.startswith(u"."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(u".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd", ".pyx"]:
            if exists(base_path + ext):
                return True
        return False 
Example 7
Project: misp42splunk   Author: remg427   File: api_documenter.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def api_get_spec(context, method_list):
    '''Generates and Returns the spec file data
    :param context: Dictionary with app, session, version and api fields
    :type: ```dict```
    :param method_list: List of API methods to call
    :type: ```list```
    :return: generated spec file
    :rtype: ```basestring```
    '''
    _generate_documentation(context, method_list)
    with open(tempfile.gettempdir() + op.sep + 'spec.yaml') as stream:
        try:
            spec_file = yaml.load(stream)
        except yaml.YAMLError as ex:
            raise Exception("Please try again. Exception: {}".format(ex))
        return json.dumps(spec_file) 
Example 8
Project: misp42splunk   Author: remg427   File: api_documenter.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def write_temp(self):
        '''
        Stores changes to the spec in a temp file.
        '''
        spec = {
            "swagger": self.api.__getattribute__('swagger'),
            "info": self.api.__getattribute__('info'),
            "host": self.api.__getattribute__('host'),
            "schemes": self.api.__getattribute__('schemes'),
            "consumes": self.api.__getattribute__('consumes'),
            "produces": self.api.__getattribute__('produces'),
            "paths": self.api.__getattribute__('paths'),
            "definitions": self.api.__getattribute__('definitions')
        }

        stream = file((tempfile.gettempdir() + op.sep + 'temp.yaml'), 'w')
        for x in self.order:
            yaml.dump({x: spec[x]}, stream, default_flow_style=False) 
Example 9
Project: misp42splunk   Author: remg427   File: api_documenter.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def update_spec(self):
        '''
        Updates the specification from the temp file.
        '''
        try:
            os.rename(
                tempfile.gettempdir() +
                op.sep +
                'temp.yaml',
                tempfile.gettempdir() +
                op.sep +
                'spec.yaml')
        except Exception as e:
            raise Exception(
                "Spec file not found, please try again."
                " Exception: {}".format(e)) 
Example 10
Project: codimension   Author: SergeySatskiy   File: globals.py    GNU General Public License v3.0 6 votes vote down vote up
def getSubdirs(path, baseNamesOnly=True, excludePythonModulesDirs=True):
    """Provides a list of sub directories for the given path"""
    subdirs = []
    try:
        path = realpath(path) + sep
        for item in os.listdir(path):
            candidate = path + item
            if isdir(candidate):
                if excludePythonModulesDirs:
                    modFile = candidate + sep + "__init__.py"
                    if exists(modFile):
                        continue
                if baseNamesOnly:
                    subdirs.append(item)
                else:
                    subdirs.append(candidate)
    except:
        pass
    return subdirs 
Example 11
Project: codimension   Author: SergeySatskiy   File: findinfiles.py    GNU General Public License v3.0 6 votes vote down vote up
def __matchInDir(path, filters, startTime):
        """Provides the 'match' and 'too long' statuses"""
        matched = False
        tooLong = False
        for item in listdir(path):
            if time.time() - startTime > 0.1:
                tooLong = True
                return matched, tooLong
            if isdir(path + item):
                dname = path + item + sep
                matched, tooLong = FindInFilesDialog.__matchInDir(dname,
                                                                  filters,
                                                                  startTime)
                if matched or tooLong:
                    return matched, tooLong
                continue
            if FindInFilesDialog.__filterMatch(filters, path + item):
                matched = True
                return matched, tooLong
        return matched, tooLong 
Example 12
Project: codimension   Author: SergeySatskiy   File: findinfiles.py    GNU General Public License v3.0 6 votes vote down vote up
def __projectFiles(self, filters):
        """Project files list respecting the mask"""
        mainWindow = GlobalData().mainWindow
        files = []
        for fname in GlobalData().project.filesList:
            if fname.endswith(sep):
                continue
            if self.__filterMatch(filters, fname):
                widget = mainWindow.getWidgetForFileName(fname)
                if widget is None:
                    # Do not check for broken symlinks
                    if isFileSearchable(fname, False):
                        files.append(ItemToSearchIn(fname, ""))
                else:
                    if widget.getType() in \
                                [MainWindowTabWidgetBase.PlainTextEditor]:
                        files.append(ItemToSearchIn(fname,
                                                    widget.getUUID()))
            QApplication.processEvents()
            if self.__cancelRequest:
                raise Exception("Cancel request")
        return files 
Example 13
Project: TESTGIT   Author: B-ROY   File: _os.py    Apache License 2.0 6 votes vote down vote up
def safe_join(base, *paths):
    """
    Joins one or more path components to the base path component intelligently.
    Returns a normalized, absolute version of the final path.

    The final path must be located inside of the base path component (otherwise
    a ValueError is raised).
    """
    base = force_text(base)
    paths = [force_text(p) for p in paths]
    final_path = abspathu(join(base, *paths))
    base_path = abspathu(base)
    # Ensure final_path starts with base_path (using normcase to ensure we
    # don't false-negative on case insensitive operating systems like Windows),
    # further, one of the following conditions must be true:
    #  a) The next character is the path separator (to prevent conditions like
    #     safe_join("/dir", "/../d"))
    #  b) The final path must be the same as the base path.
    #  c) The base path must be the most root path (meaning either "/" or "C:\\")
    if (not normcase(final_path).startswith(normcase(base_path + sep)) and
        normcase(final_path) != normcase(base_path) and
        dirname(normcase(base_path)) != normcase(base_path)):
        raise ValueError('The joined path (%s) is located outside of the base '
                         'path component (%s)' % (final_path, base_path))
    return final_path 
Example 14
Project: AneMo   Author: jspargo   File: _os.py    GNU General Public License v2.0 6 votes vote down vote up
def safe_join(base, *paths):
    """
    Joins one or more path components to the base path component intelligently.
    Returns a normalized, absolute version of the final path.

    The final path must be located inside of the base path component (otherwise
    a ValueError is raised).
    """
    base = force_text(base)
    paths = [force_text(p) for p in paths]
    final_path = abspathu(join(base, *paths))
    base_path = abspathu(base)
    # Ensure final_path starts with base_path (using normcase to ensure we
    # don't false-negative on case insensitive operating systems like Windows),
    # further, one of the following conditions must be true:
    #  a) The next character is the path separator (to prevent conditions like
    #     safe_join("/dir", "/../d"))
    #  b) The final path must be the same as the base path.
    #  c) The base path must be the most root path (meaning either "/" or "C:\\")
    if (not normcase(final_path).startswith(normcase(base_path + sep)) and
            normcase(final_path) != normcase(base_path) and
            dirname(normcase(base_path)) != normcase(base_path)):
        raise ValueError('The joined path (%s) is located outside of the base '
                         'path component (%s)' % (final_path, base_path))
    return final_path 
Example 15
Project: pybids   Author: bids-standard   File: test_transformations.py    MIT License 6 votes vote down vote up
def test_orthogonalize_dense(collection):
    transform.Factor(collection, 'trial_type', sep=sep)

    sampling_rate = collection.sampling_rate
    # Store pre-orth variables needed for tests
    pg_pre = collection['trial_type/parametric gain'].to_dense(sampling_rate)
    rt = collection['RT'].to_dense(sampling_rate)

    # Orthogonalize and store result
    transform.Orthogonalize(collection, variables='trial_type/parametric gain',
                            other='RT', dense=True, groupby=['run', 'subject'])
    pg_post = collection['trial_type/parametric gain']

    # Verify that the to_dense() calls result in identical indexing
    ent_cols = ['subject', 'run']
    assert pg_pre.to_df()[ent_cols].equals(rt.to_df()[ent_cols])
    assert pg_post.to_df()[ent_cols].equals(rt.to_df()[ent_cols])

    vals = np.c_[rt.values, pg_pre.values, pg_post.values]
    df = pd.DataFrame(vals, columns=['rt', 'pre', 'post'])
    groupby = rt.get_grouper(['run', 'subject'])
    pre_r = df.groupby(groupby).apply(lambda x: x.corr().iloc[0, 1])
    post_r = df.groupby(groupby).apply(lambda x: x.corr().iloc[0, 2])
    assert (pre_r > 0.2).any()
    assert (post_r < 0.0001).all() 
Example 16
Project: pyblish-win   Author: pyblish   File: fix_import.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def probably_a_local_import(self, imp_name):
        if imp_name.startswith(u"."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(u".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
            if exists(base_path + ext):
                return True
        return False 
Example 17
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 18
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in os.walk(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 19
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 20
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in os.walk(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 21
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: loaders.py    MIT License 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 22
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: loaders.py    MIT License 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 23
Project: spleeter   Author: deezer   File: dataset.py    MIT License 5 votes vote down vote up
def expand_path(self, sample):
        """ Expands audio paths for the given sample. """
        return dict(sample, **{f'{instrument}_path': tf.string_join(
            (self._audio_path, sample[f'{instrument}_path']), SEPARATOR)
            for instrument in self._instruments}) 
Example 24
Project: flasky   Author: RoseOu   File: loaders.py    MIT License 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 25
Project: flasky   Author: RoseOu   File: loaders.py    MIT License 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in os.walk(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 26
Project: mealpy   Author: edmundmok   File: venv_update.py    MIT License 5 votes vote down vote up
def timid_relpath(arg):
    """convert an argument to a relative path, carefully"""
    # TODO-TEST: unit tests
    from os.path import isabs, relpath, sep
    if isabs(arg):
        result = relpath(arg)
        if result.count(sep) + 1 < arg.count(sep):
            return result

    return arg 
Example 27
Project: specio   Author: paris-saclay-cds   File: test_request.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def test_request():
    filename = 'file://' + join(DATA_PATH, 'data', 'spectra.foobar')
    R = Request(filename)
    assert R._uri_type == core.request.URI_FILENAME
    assert R.get_local_filename() == filename[7:]
    file_obj = R.get_file()
    assert file_obj.name == filename[7:]

    filename = join(DATA_PATH, 'data', 'spectra.foobar')
    R = Request(filename)
    assert R._uri_type == core.request.URI_FILENAME
    assert R.get_local_filename() == filename
    file_obj = R.get_file()
    assert file_obj.name == filename

    filename = join(DATA_PATH, 'data', 'spectra.foobar')
    file_obj = open(filename, 'rb')
    R = Request(file_obj)
    assert R.get_local_filename() == filename
    assert R.get_file().name == filename

    shutil.copy(filename, expanduser('~/'))
    filename = '~/spectra.foobar'
    R = Request(filename)
    assert R.filename == expanduser('~/spectra.foobar').replace('/', sep)
    os.remove(expanduser(filename))
    filename = join(DATA_PATH, 'data', 'spectra.foobar')
    R = Request(filename, some_kwarg='something')
    assert R.kwargs == {'some_kwarg': 'something'} 
Example 28
Project: calmjs   Author: calmjs   File: base.py    GNU General Public License v2.0 5 votes vote down vote up
def which_with_node_modules(self):
        """
        Which with node_path and node_modules
        """

        if self.binary is None:
            return None

        # first, log down the pedantic things...
        if isdir(self.join_cwd(NODE_MODULES)):
            logger.debug(
                "'%s' instance will attempt to locate '%s' binary from "
                "%s%s%s%s%s, located through the working directory",
                self.__class__.__name__, self.binary,
                self.join_cwd(), sep, NODE_MODULES, sep, NODE_MODULES_BIN,
            )
        if self.node_path:
            logger.debug(
                "'%s' instance will attempt to locate '%s' binary from "
                "its %s of %s",
                self.__class__.__name__, self.binary,
                NODE_PATH, self.node_path,
            )

        paths = self.find_node_modules_basedir()
        whichpaths = pathsep.join(join(p, NODE_MODULES_BIN) for p in paths)

        if paths:
            logger.debug(
                "'%s' instance located %d possible paths to the '%s' binary, "
                "which are %s",
                self.__class__.__name__, len(paths), self.binary, whichpaths,
            )

        return which(self.binary, path=whichpaths) 
Example 29
Project: imgcomp-cvpr   Author: fab-jul   File: val_images.py    GNU General Public License v3.0 5 votes vote down vote up
def get_path_component_before_glob(p):
    """ Given some path ending in one or more components containing *, return the left-most non-empty component not
    containig *, e.g., /some/path/dir/*/*/*.png => dir """
    for comp in reversed(p.strip(path.sep).split(path.sep)):
        if '*' not in comp:
            return comp
    raise ValueError('No component without *: {}'.format(p)) 
Example 30
Project: L3C-PyTorch   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 5 votes vote down vote up
def create_unique_log_dir(config_rel_paths, log_dir_root, line_breaking_chars_pat=r'[-]',
                          postfix=None, restore_dir=None, strip_ext=None):
    """
    0117_1704 [email protected]_med_8e*5_deePer_b50_noHM_C16 [email protected]_res_shallow [email protected]_1340
    :param config_rel_paths: paths to the configs, relative to the config root dir
    :param log_dir_root: In this directory, all log dirs are stored. Created if needed.
    :param line_breaking_chars_pat:
    :param postfix: appended to the returned log dir
    :param restore_dir: if given, expected to be a log dir. the JOB_ID of that will be appended
    :param strip_ext: if given, do not store extension `strip_ext` of config_rel_paths
    :return: path to a newly created directory
    """
    if any('@' in config_rel_path for config_rel_path in config_rel_paths):
        raise ValueError('"@" not allowed in paths, got {}'.format(config_rel_paths))

    if strip_ext:
        assert all(strip_ext in c for c in config_rel_paths)
        config_rel_paths = [c.replace(strip_ext, '') for c in config_rel_paths]

    def prep_path(p):
        p = p.replace(path.sep, '@')
        return re.sub(line_breaking_chars_pat, '*', p)

    postfix_dir_name = ' '.join(map(prep_path, config_rel_paths))
    if restore_dir:
        _, restore_job_component = _split_log_dir(restore_dir)
        restore_job_id = log_date_from_log_dir(restore_job_component)
        postfix_dir_name += ' {restore_prefix}{job_id}'.format(
                restore_prefix=_RESTORE_PREFIX, job_id=restore_job_id)
    if postfix:
        if isinstance(postfix, list):
            postfix = ' '.join(postfix)
        postfix_dir_name += ' ' + postfix
    return _mkdir_threadsafe_unique(log_dir_root, datetime.now(), postfix_dir_name) 
Example 31
Project: L3C-PyTorch   Author: fab-jul   File: logdir_helpers.py    GNU General Public License v3.0 5 votes vote down vote up
def parse_log_dir(log_dir, configs_dir, base_dirs, append_ext=''):
    """
    Given a log_dir produced by `create_unique_log_dir`, return the full paths of all configs used.
    The log dir has thus the following format
            {now} {netconfig} {probconfig} [[email protected]_YYYY] [{postfix} {postfix}]

    :param log_dir: the log dir to parse
    :param configs_dir: the root config dir, where all the configs live
    :param base_dirs: Prefixed to the paths of the configs, e.g., ['ae', 'pc']
    :return: all config paths, as well as the postfix if one was given
    """
    base_dirs = [path.join(configs_dir, base_dir) for base_dir in base_dirs]
    log_dir = path.basename(log_dir.strip(path.sep))

    comps = log_dir.split(' ')
    assert is_log_date(comps[0]), 'Invalid log_dir: {}'.format(log_dir)

    assert len(comps) > len(base_dirs), 'Expected a base dir for every component, got {} and {}'.format(
            comps, base_dirs)
    config_components = comps[1:(1+len(base_dirs))]
    has_restore = any(_RESTORE_PREFIX in c for c in comps)
    postfix = comps[1+len(base_dirs)+has_restore:]

    def get_real_path(base, prepped_p):
        p_glob = prepped_p.replace('@', path.sep)
        p_glob = path.join(base, p_glob) + append_ext  # e.g., ae_configs/p_glob.cf
        glob_matches = glob.glob(p_glob)
        # We always only replace one character with *, so filter for those.
        # I.e. lr1e-5 will become lr1e*5, which will match lr1e-5 but also lr1e-4.5
        glob_matches_of_same_len = [g for g in glob_matches if len(g) == len(p_glob)]
        if len(glob_matches_of_same_len) != 1:
            raise ValueError('Cannot find config on disk: {} (matches: {})'.format(p_glob, glob_matches_of_same_len))
        return glob_matches_of_same_len[0]

    return LogDirComps(
            config_paths=tuple(get_real_path(base_dir, comp)
                               for base_dir, comp in zip(base_dirs, config_components)),
            postfix=tuple(postfix) if postfix else None)


# ------------------------------------------------------------------------------ 
Example 32
Project: vscode-mayapy   Author: FXTD-ODYSSEY   File: fix_import.py    MIT License 5 votes vote down vote up
def probably_a_local_import(self, imp_name):
        if imp_name.startswith(u"."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(u".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
            if exists(base_path + ext):
                return True
        return False 
Example 33
Project: misp42splunk   Author: remg427   File: loaders.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 34
Project: chattR   Author: patrickstocklin   File: _os.py    GNU General Public License v2.0 5 votes vote down vote up
def safe_join(base, *paths):
    """
    Joins one or more path components to the base path component intelligently.
    Returns a normalized, absolute version of the final path.

    The final path must be located inside of the base path component (otherwise
    a ValueError is raised).
    """
    base = force_text(base)
    paths = [force_text(p) for p in paths]
    final_path = abspathu(join(base, *paths))
    base_path = abspathu(base)
    # Ensure final_path starts with base_path (using normcase to ensure we
    # don't false-negative on case insensitive operating systems like Windows),
    # further, one of the following conditions must be true:
    #  a) The next character is the path separator (to prevent conditions like
    #     safe_join("/dir", "/../d"))
    #  b) The final path must be the same as the base path.
    #  c) The base path must be the most root path (meaning either "/" or "C:\\")
    if (not normcase(final_path).startswith(normcase(base_path + sep)) and
            normcase(final_path) != normcase(base_path) and
            dirname(normcase(base_path)) != normcase(base_path)):
        raise SuspiciousFileOperation(
            'The joined path ({}) is located outside of the base path '
            'component ({})'.format(final_path, base_path))
    return final_path 
Example 35
Project: Jtyoui   Author: jtyoui   File: __init__.py    MIT License 5 votes vote down vote up
def load_zip(zip_name, file_name, encoding='UTF-8', sep='\n'):
    """加载zip数据

    :param zip_name: 压缩包的名字
    :param file_name: 压缩包里面文件的名字
    :param encoding: 文件的编码
    :param sep: 压缩文件里面的换行符
    :return: 压缩包里面的数据:默认编码的UTF-8
    """
    file_zip = path.join(file_zip_path, zip_name)
    f = zipfile.ZipFile(file_zip)
    fp = f.read(file_name)
    lines = fp.decode(encoding).split(sep)
    return lines 
Example 36
Project: TornadoWeb   Author: VxCoder   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 37
Project: TornadoWeb   Author: VxCoder   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 38
Project: Flask_Blog   Author: sugarguo   File: loaders.py    GNU General Public License v3.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 39
Project: Flask_Blog   Author: sugarguo   File: loaders.py    GNU General Public License v3.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 40
Project: LaserTOF   Author: kyleuckert   File: test_misc_util.py    MIT License 5 votes vote down vote up
def test_1(self):
        n = lambda path: path.replace('/', sep)
        assert_equal(minrelpath(n('aa/bb')), n('aa/bb'))
        assert_equal(minrelpath('..'), '..')
        assert_equal(minrelpath(n('aa/..')), '')
        assert_equal(minrelpath(n('aa/../bb')), 'bb')
        assert_equal(minrelpath(n('aa/bb/..')), 'aa')
        assert_equal(minrelpath(n('aa/bb/../..')), '')
        assert_equal(minrelpath(n('aa/bb/../cc/../dd')), n('aa/dd'))
        assert_equal(minrelpath(n('.././..')), n('../..'))
        assert_equal(minrelpath(n('aa/bb/.././../dd')), n('dd')) 
Example 41
Project: NiujiaoDebugger   Author: MrSrc   File: fix_import.py    GNU General Public License v3.0 5 votes vote down vote up
def probably_a_local_import(self, imp_name):
        if imp_name.startswith("."):
            # Relative imports are certainly not local imports.
            return False
        imp_name = imp_name.split(".", 1)[0]
        base_path = dirname(self.filename)
        base_path = join(base_path, imp_name)
        # If there is no __init__.py next to the file its not in a package
        # so can't be a relative import.
        if not exists(join(dirname(base_path), "__init__.py")):
            return False
        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
            if exists(base_path + ext):
                return True
        return False 
Example 42
Project: Health-Checker   Author: KriAga   File: loaders.py    MIT License 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 43
Project: Health-Checker   Author: KriAga   File: loaders.py    MIT License 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 44
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_misc_util.py    MIT License 5 votes vote down vote up
def test_1(self):
        n = lambda path: path.replace('/', sep)
        assert_equal(minrelpath(n('aa/bb')), n('aa/bb'))
        assert_equal(minrelpath('..'), '..')
        assert_equal(minrelpath(n('aa/..')), '')
        assert_equal(minrelpath(n('aa/../bb')), 'bb')
        assert_equal(minrelpath(n('aa/bb/..')), 'aa')
        assert_equal(minrelpath(n('aa/bb/../..')), '')
        assert_equal(minrelpath(n('aa/bb/../cc/../dd')), n('aa/dd'))
        assert_equal(minrelpath(n('.././..')), n('../..'))
        assert_equal(minrelpath(n('aa/bb/.././../dd')), n('dd')) 
Example 45
Project: recruit   Author: Frank-qlu   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 46
Project: recruit   Author: Frank-qlu   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 47
Project: recruit   Author: Frank-qlu   File: test_misc_util.py    Apache License 2.0 5 votes vote down vote up
def test_1(self):
        n = lambda path: path.replace('/', sep)
        assert_equal(minrelpath(n('aa/bb')), n('aa/bb'))
        assert_equal(minrelpath('..'), '..')
        assert_equal(minrelpath(n('aa/..')), '')
        assert_equal(minrelpath(n('aa/../bb')), 'bb')
        assert_equal(minrelpath(n('aa/bb/..')), 'aa')
        assert_equal(minrelpath(n('aa/bb/../..')), '')
        assert_equal(minrelpath(n('aa/bb/../cc/../dd')), n('aa/dd'))
        assert_equal(minrelpath(n('.././..')), n('../..'))
        assert_equal(minrelpath(n('aa/bb/.././../dd')), n('dd')) 
Example 48
Project: recruit   Author: Frank-qlu   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if path.sep in piece \
           or (path.altsep and path.altsep in piece) or \
           piece == path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces 
Example 49
Project: recruit   Author: Frank-qlu   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            walk_dir = os.walk(searchpath, followlinks=self.followlinks)
            for dirpath, dirnames, filenames in walk_dir:
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 50
Project: dcatd   Author: Amsterdam   File: app_test.py    Mozilla Public License 2.0 5 votes vote down vote up
def testUpload(self):
        headers = {
            'Accept': "application/json",
            'Authorization': self.redact_token,
            'Cache-Control': "no-cache",
        }

        async def returner(value):
            return value

        client = self.client

        when(swift)._put_file_to_object_store(
            any(str),
            'application/json',
            any,
            filename='test_upload.json').thenReturn(returner("randomness"))

        with open(self._WORKING_PATH + path.sep + 'test.json', 'rb') as file:
            data = FormData()
            data.add_field('distribution', file,
                           filename='test_upload.json',
                           content_type='application/json')

            response = await client.post(
                path='/files',
                data=data,
                headers=headers)

        unstub()

        self.assertEqual(response.status, 201, 'File upload mislukt')