Python sys.getfilesystemencoding() Examples

def _rename(src, dst):
            '''
            if not isinstance(src, unicode):
                src = unicode(src, sys.getfilesystemencoding())
            if not isinstance(dst, unicode):
                dst = unicode(dst, sys.getfilesystemencoding())
            '''
            if _rename_atomic(src, dst):
                return True
            retry = 0
            rv = False
            while not rv and retry < 100:
                rv = _MoveFileEx(src, dst, _MOVEFILE_REPLACE_EXISTING |
                                           _MOVEFILE_WRITE_THROUGH)
                if not rv:
                    time.sleep(0.001)
                    retry += 1
            return rv

        # new in Vista and Windows Server 2008 

def _initial_load():
  global _CUR_CONTEXT
  to_assign = {}

  ctx_path = os.environ.get(ENV_KEY)
  if ctx_path:
    if six.PY2:
      ctx_path = ctx_path.decode(sys.getfilesystemencoding())
    _LOGGER.debug('Loading LUCI_CONTEXT: %r', ctx_path)
    try:
      with open(ctx_path, 'r') as f:
        loaded = _to_utf8(json.load(f))
        if _check_ok(loaded):
          to_assign = loaded
    except OSError as ex:
      _LOGGER.error('LUCI_CONTEXT failed to open: %s', ex)
    except IOError as ex:
      _LOGGER.error('LUCI_CONTEXT failed to read: %s', ex)
    except ValueError as ex:
      _LOGGER.error('LUCI_CONTEXT failed to decode: %s', ex)

  _CUR_CONTEXT = to_assign 

def read_history(self):
        if fsutils.isfile(self.history_file):
            fp = fsutils.get_fileptr(self.history_file)
            lines = [line.strip(' \n\r') for line in fp.readlines()]
            for line in lines:
                items = line.split('\t')
                if len(items) == 3:
                    status = int(items[0])
                    path = items[1]
                    try:
                        path = path.decode('utf-8')
                    except Exception:
                        path = path.decode(sys.getfilesystemencoding())
                    finally:
                        path = path.encode('utf-8')
                    timestamp = int(items[2])
                    self.history.append([status, path, timestamp])
            fp.close() 

def setPaths():
    """
    Sets absolute paths for project directories and files
    """

    paths.POCSUITE_DATA_PATH = os.path.join(paths.POCSUITE_ROOT_PATH, "data")

    paths.USER_AGENTS = os.path.join(paths.POCSUITE_DATA_PATH, "user-agents.txt")
    paths.WEAK_PASS = os.path.join(paths.POCSUITE_DATA_PATH, "password-top100.txt")
    paths.LARGE_WEAK_PASS = os.path.join(paths.POCSUITE_DATA_PATH, "password-top1000.txt")

    _ = os.path.join(os.path.expanduser("~"), ".pocsuite")
    paths.POCSUITE_OUTPUT_PATH = getUnicode(paths.get("POCSUITE_OUTPUT_PATH", os.path.join(_, "output")), encoding=sys.getfilesystemencoding())

    paths.POCSUITE_MODULES_PATH = os.path.join(_, "modules")
    paths.POCSUITE_TMP_PATH = os.path.join(paths.POCSUITE_MODULES_PATH, "tmp")
    paths.POCSUITE_HOME_PATH = os.path.expanduser("~")
    paths.POCSUITE_RC_PATH = paths.POCSUITE_HOME_PATH + "/.pocsuiterc" 

def get_filesystem_encoding():
    """Returns the filesystem encoding that should be used. Note that this is
    different from the Python understanding of the filesystem encoding which
    might be deeply flawed. Do not use this value against Python's unicode APIs
    because it might be different. See :ref:`filesystem-encoding` for the exact
    behavior.

    The concept of a filesystem encoding in generally is not something you
    should rely on. As such if you ever need to use this function except for
    writing wrapper code reconsider.
    """
    global _warned_about_filesystem_encoding
    rv = sys.getfilesystemencoding()
    if has_likely_buggy_unicode_filesystem and not rv or _is_ascii_encoding(rv):
        if not _warned_about_filesystem_encoding:
            warnings.warn(
                "Detected a misconfigured UNIX filesystem: Will use"
                " UTF-8 as filesystem encoding instead of {0!r}".format(rv),
                BrokenFilesystemWarning,
            )
            _warned_about_filesystem_encoding = True
        return "utf-8"
    return rv 

def fix_script(path):
    """Replace #!python with #!/path/to/python
    Return True if file was changed."""
    # XXX RECORD hashes will need to be updated
    if os.path.isfile(path):
        with open(path, 'rb') as script:
            firstline = script.readline()
            if not firstline.startswith(b'#!python'):
                return False
            exename = sys.executable.encode(sys.getfilesystemencoding())
            firstline = b'#!' + exename + os.linesep.encode("ascii")
            rest = script.read()
        with open(path, 'wb') as script:
            script.write(firstline)
            script.write(rest)
        return True 

def setup_py(self):
        assert self.source_dir, "No source dir for %s" % self
        try:
            import setuptools  # noqa
        except ImportError:
            if get_installed_version('setuptools') is None:
                add_msg = "Please install setuptools."
            else:
                add_msg = traceback.format_exc()
            # Setuptools is not available
            raise InstallationError(
                "Could not import setuptools which is required to "
                "install from a source distribution.\n%s" % add_msg
            )

        setup_py = os.path.join(self.setup_py_dir, 'setup.py')

        # Python2 __file__ should not be unicode
        if six.PY2 and isinstance(setup_py, six.text_type):
            setup_py = setup_py.encode(sys.getfilesystemencoding())

        return setup_py 

def make_dokuwiki_pagename(mediawiki_name):
    """
    Convert a canonical mediawiki pagename to a dokuwiki pagename

    Any namespacing that is in the form of a / is replaced with a :
    """
    result = mediawiki_name.replace(" ","_")
    # We have pages that have ':' in them - replace with underscores
    result = result.replace(':', '_')
    result = names.clean_id(camel_to_underscore(result)).replace("/",":")
    # Some of our mediawiki page names begin with a '/', which results in os.path.join assuming the page is an absolute path.
    if result[0] == ':':
      result = result.lstrip(':')
    # Fix any pages that began with a space, because that breaks dokuwiki
    result = result.replace(":_", ":")
    result = codecs.encode(result, sys.getfilesystemencoding(), "replace")
    return result 

def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE, newlines=True):
    # Linux py2.7 encode as list without quotes no empty element for parameters
    # linux py3.x no encode and as list without quotes no empty element for parameters
    # windows py2.7 encode as string with quotes empty element for parameters is okay
    # windows py 3.x no encode and as string with quotes empty element for parameters is okay
    # separate handling for windows and linux
    if os.name == 'nt':
        for key, element in enumerate(command):
            if key in quotes:
                command[key] = '"' + element + '"'
        exc_command = " ".join(command)
        if sys.version_info < (3, 0):
            exc_command = exc_command.encode(sys.getfilesystemencoding())
    else:
        if sys.version_info < (3, 0):
            exc_command = [x.encode(sys.getfilesystemencoding()) for x in command]
        else:
            exc_command = [x for x in command]

    return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=newlines, env=env) 

def check_unrar(unrarLocation):
    if not unrarLocation:
        return

    if not os.path.exists(unrarLocation):
        return _('Unrar binary file not found')

    try:
        if sys.version_info < (3, 0):
            unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
        unrarLocation = [unrarLocation]
        for lines in process_wait(unrarLocation):
            value = re.search('UNRAR (.*) freeware', lines, re.IGNORECASE)
            if value:
                version = value.group(1)
                log.debug("unrar version %s", version)
                break
    except (OSError, UnicodeDecodeError) as err:
        log.exception(err)
        return _('Error excecuting UnRar') 

def fix_script(path):
    """Replace #!python with #!/path/to/python
    Return True if file was changed."""
    # XXX RECORD hashes will need to be updated
    if os.path.isfile(path):
        with open(path, 'rb') as script:
            firstline = script.readline()
            if not firstline.startswith(b'#!python'):
                return False
            exename = sys.executable.encode(sys.getfilesystemencoding())
            firstline = b'#!' + exename + os.linesep.encode("ascii")
            rest = script.read()
        with open(path, 'wb') as script:
            script.write(firstline)
            script.write(rest)
        return True 

def fsencode(filename):
        if isinstance(filename, bytes):
            return filename
        elif isinstance(filename, str):
            return filename.encode(sys.getfilesystemencoding())
        else:
            raise TypeError("expect bytes or str, not %s" %
                            type(filename).__name__) 

def setup_py(self):
        assert self.source_dir, "No source dir for %s" % self
        try:
            import setuptools  # noqa
        except ImportError:
            if get_installed_version('setuptools') is None:
                add_msg = "Please install setuptools."
            else:
                add_msg = traceback.format_exc()
            # Setuptools is not available
            raise InstallationError(
                "Could not import setuptools which is required to "
                "install from a source distribution.\n%s" % add_msg
            )

        setup_py = os.path.join(self.setup_py_dir, 'setup.py')

        # Python2 __file__ should not be unicode
        if six.PY2 and isinstance(setup_py, six.text_type):
            setup_py = setup_py.encode(sys.getfilesystemencoding())

        return setup_py 

def get_filesystem_encoding():
    """
    Returns the filesystem encoding that should be used. Note that this is
    different from the Python understanding of the filesystem encoding which
    might be deeply flawed. Do not use this value against Python's unicode APIs
    because it might be different. See :ref:`filesystem-encoding` for the exact
    behavior.

    The concept of a filesystem encoding in generally is not something you
    should rely on. As such if you ever need to use this function except for
    writing wrapper code reconsider.
    """
    global _warned_about_filesystem_encoding
    rv = sys.getfilesystemencoding()
    if has_likely_buggy_unicode_filesystem and not rv \
       or _is_ascii_encoding(rv):
        if not _warned_about_filesystem_encoding:
            warnings.warn(
                'Detected a misconfigured UNIX filesystem: Will use UTF-8 as '
                'filesystem encoding instead of {0!r}'.format(rv),
                BrokenFilesystemWarning)
            _warned_about_filesystem_encoding = True
        return 'utf-8'
    return rv 

def _safe_path(self, path):
        enc_warn = "'%s' not %s encodable -- skipping"

        # To avoid accidental trans-codings errors, first to unicode
        u_path = unicode_utils.filesys_decode(path)
        if u_path is None:
            log.warn("'%s' in unexpected encoding -- skipping" % path)
            return False

        # Must ensure utf-8 encodability
        utf8_path = unicode_utils.try_encode(u_path, "utf-8")
        if utf8_path is None:
            log.warn(enc_warn, path, 'utf-8')
            return False

        try:
            # accept is either way checks out
            if os.path.exists(u_path) or os.path.exists(utf8_path):
                return True
        # this will catch any encode errors decoding u_path
        except UnicodeEncodeError:
            log.warn(enc_warn, path, sys.getfilesystemencoding()) 

def __init__(self, name, loadfile):
        """
        Load a single plugin
        :param name: module name
        :param loadfile: the main .py file
        :raises Exception: Typically ImportError or OSError
        """

        self.name = name	# Display name.
        self.folder = name	# basename of plugin folder. None for internal plugins.
        self.module = None	# None for disabled plugins.

        if loadfile:
            sys.stdout.write(('loading plugin %s from "%s"\n' % (name.replace('.', '_'), loadfile)).encode('utf-8'))
            with open(loadfile, 'rb') as plugfile:
                module = imp.load_module('plugin_%s' % name.encode('ascii', 'replace').replace('.', '_'), plugfile, loadfile.encode(sys.getfilesystemencoding()),
                                         ('.py', 'r', imp.PY_SOURCE))
                if module.plugin_start.func_code.co_argcount == 0:
                    newname = module.plugin_start()
                else:
                    newname = module.plugin_start(os.path.dirname(loadfile))
                self.name = newname and unicode(newname) or name
                self.module = module
        else:
            sys.stdout.write('plugin %s disabled\n' % name) 

def fix_script(path):
    """Replace #!python with #!/path/to/python
    Return True if file was changed."""
    # XXX RECORD hashes will need to be updated
    if os.path.isfile(path):
        with open(path, 'rb') as script:
            firstline = script.readline()
            if not firstline.startswith(b'#!python'):
                return False
            exename = sys.executable.encode(sys.getfilesystemencoding())
            firstline = b'#!' + exename + os.linesep.encode("ascii")
            rest = script.read()
        with open(path, 'wb') as script:
            script.write(firstline)
            script.write(rest)
        return True 

def setup_py(self):
        assert self.source_dir, "No source dir for %s" % self
        try:
            import setuptools  # noqa
        except ImportError:
            if get_installed_version('setuptools') is None:
                add_msg = "Please install setuptools."
            else:
                add_msg = traceback.format_exc()
            # Setuptools is not available
            raise InstallationError(
                "Could not import setuptools which is required to "
                "install from a source distribution.\n%s" % add_msg
            )

        setup_py = os.path.join(self.setup_py_dir, 'setup.py')

        # Python2 __file__ should not be unicode
        if six.PY2 and isinstance(setup_py, six.text_type):
            setup_py = setup_py.encode(sys.getfilesystemencoding())

        return setup_py 

def _check_unicode_filesystem(tmpdir):
    filename = tmpdir / ntou('☃', 'utf-8')
    tmpl = 'File system encoding ({encoding}) cannot support unicode filenames'
    msg = tmpl.format(encoding=sys.getfilesystemencoding())
    try:
        io.open(str(filename), 'w').close()
    except UnicodeEncodeError:
        pytest.skip(msg) 

def ensure_byte_string(value):
    """
    Return the given ``value`` as bytestring.

    If the given ``value`` is not a byte string, but a real unicode string, it
    is encoded with the filesystem encoding (as in
    :func:`sys.getfilesystemencoding()`).
    """
    if not isinstance(value, bytes):
        value = value.encode(sys.getfilesystemencoding())
    return value 

def ensure_unicode_string(value):
    """
    Return the given ``value`` as unicode string.

    If the given ``value`` is not a unicode string, but a byte string, it is
    decoded with the filesystem encoding (as in
    :func:`sys.getfilesystemencoding()`).
    """
    if not isinstance(value, six.text_type):
        value = value.decode(sys.getfilesystemencoding())
    return value 

def _str_to_unicode(x):
        """Handle text decoding. Internal use only"""
        if not isinstance(x, text_type):
            return x.decode(sys.getfilesystemencoding())
        return x 

def _createenviron():
    if name in ('os2', 'nt'):
        # Where Env Var Names Must Be UPPERCASE
        def check_str(value):
            if not isinstance(value, str):
                raise TypeError("str expected, not %s" % type(value).__name__)
            return value
        encode = check_str
        decode = str
        def encodekey(key):
            return encode(key).upper()
        data = {}
        for key, value in environ.items():
            data[encodekey(key)] = value
    else:
        # Where Env Var Names Can Be Mixed Case
        encoding = sys.getfilesystemencoding()
        def encode(value):
            if not isinstance(value, str):
                raise TypeError("str expected, not %s" % type(value).__name__)
            return value.encode(encoding, 'surrogateescape')
        def decode(value):
            return value.decode(encoding, 'surrogateescape')
        encodekey = encode
        data = environ
    return _Environ(data,
        encodekey, decode,
        encode, decode,
        _putenv, _unsetenv)

# unicode environ 

def _fscodec():
    encoding = sys.getfilesystemencoding()
    if encoding == 'mbcs':
        errors = 'strict'
    else:
        errors = 'surrogateescape'

    def fsencode(filename):
        """
        Encode filename to the filesystem encoding with 'surrogateescape' error
        handler, return bytes unchanged. On Windows, use 'strict' error handler if
        the file system encoding is 'mbcs' (which is the default encoding).
        """
        if isinstance(filename, bytes):
            return filename
        elif isinstance(filename, str):
            return filename.encode(encoding, errors)
        else:
            raise TypeError("expect bytes or str, not %s" % type(filename).__name__)

    def fsdecode(filename):
        """
        Decode filename from the filesystem encoding with 'surrogateescape' error
        handler, return str unchanged. On Windows, use 'strict' error handler if
        the file system encoding is 'mbcs' (which is the default encoding).
        """
        if isinstance(filename, str):
            return filename
        elif isinstance(filename, bytes):
            return filename.decode(encoding, errors)
        else:
            raise TypeError("expect bytes or str, not %s" % type(filename).__name__)

    return fsencode, fsdecode 

def description_of(lines, name='stdin'):
    """
    Return a string describing the probable encoding of a file or
    list of strings.

    :param lines: The lines to get the encoding of.
    :type lines: Iterable of bytes
    :param name: Name of file or collection of lines
    :type name: str
    """
    u = UniversalDetector()
    for line in lines:
        line = bytearray(line)
        u.feed(line)
        # shortcut out of the loop to save reading further - particularly useful if we read a BOM.
        if u.done:
            break
    u.close()
    result = u.result
    if PY2:
        name = name.decode(sys.getfilesystemencoding(), 'ignore')
    if result['encoding']:
        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
                                                     result['confidence'])
    else:
        return '{0}: no result'.format(name) 

def run(self, command):
    env = os.environ.copy()
    env["GOOEY"] = "1"
    self._process = subprocess.Popen(
      command.encode(sys.getfilesystemencoding()),
      bufsize=1, stdout=subprocess.PIPE,
      stderr=subprocess.STDOUT, shell=True, env=env)
    Pool(1).apply_async(self._forward_stdout, (self._process,)) 

def description_of(lines, name='stdin'):
    """
    Return a string describing the probable encoding of a file or
    list of strings.

    :param lines: The lines to get the encoding of.
    :type lines: Iterable of bytes
    :param name: Name of file or collection of lines
    :type name: str
    """
    u = UniversalDetector()
    for line in lines:
        line = bytearray(line)
        u.feed(line)
        # shortcut out of the loop to save reading further - particularly useful if we read a BOM.
        if u.done:
            break
    u.close()
    result = u.result
    if PY2:
        name = name.decode(sys.getfilesystemencoding(), 'ignore')
    if result['encoding']:
        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
                                                     result['confidence'])
    else:
        return '{0}: no result'.format(name) 

def description_of(lines, name='stdin'):
    """
    Return a string describing the probable encoding of a file or
    list of strings.

    :param lines: The lines to get the encoding of.
    :type lines: Iterable of bytes
    :param name: Name of file or collection of lines
    :type name: str
    """
    u = UniversalDetector()
    for line in lines:
        line = bytearray(line)
        u.feed(line)
        # shortcut out of the loop to save reading further - particularly useful if we read a BOM.
        if u.done:
            break
    u.close()
    result = u.result
    if PY2:
        name = name.decode(sys.getfilesystemencoding(), 'ignore')
    if result['encoding']:
        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
                                                     result['confidence'])
    else:
        return '{0}: no result'.format(name) 

def sanitize_filename(name: str,
                      replacement: typing.Optional[str] = '_') -> str:
    """Replace invalid filename characters.

    Note: This should be used for the basename, as it also removes the path
    separator.

    Args:
        name: The filename.
        replacement: The replacement character (or None).
    """
    if replacement is None:
        replacement = ''

    # Remove chars which can't be encoded in the filename encoding.
    # See https://github.com/qutebrowser/qutebrowser/issues/427
    encoding = sys.getfilesystemencoding()
    name = force_encoding(name, encoding)

    # See also
    # https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words
    if is_windows:
        bad_chars = '\\/:*?"<>|'
    elif is_mac:
        # Colons can be confusing in finder https://superuser.com/a/326627
        bad_chars = '/:'
    else:
        bad_chars = '/'

    for bad_char in bad_chars:
        name = name.replace(bad_char, replacement)
    return name 

def create_full_filename(basename, filename):
    """Create a full filename based on the given basename and filename.

    Args:
        basename: The basename to use if filename is a directory.
        filename: The path to a folder or file where you want to save.

    Return:
        The full absolute path, or None if filename creation was not possible.
    """
    basename = utils.sanitize_filename(basename)
    # Filename can be a full path so don't use sanitize_filename on it.
    # Remove chars which can't be encoded in the filename encoding.
    # See https://github.com/qutebrowser/qutebrowser/issues/427
    encoding = sys.getfilesystemencoding()
    filename = utils.force_encoding(filename, encoding)
    if os.path.isabs(filename) and (os.path.isdir(filename) or
                                    filename.endswith(os.sep)):
        # We got an absolute directory from the user, so we save it under
        # the default filename in that directory.
        return os.path.join(filename, basename)
    elif os.path.isabs(filename):
        # We got an absolute filename from the user, so we save it under
        # that filename.
        return filename
    return None