Python os.open() Examples

The following are code examples for showing how to use os.open(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: test_tempfile.py    GNU Lesser General Public License v3.0 8 votes vote down vote up
def test_usable_template(self):
        # gettempprefix returns a usable prefix string

        # Create a temp directory, avoiding use of the prefix.
        # Then attempt to create a file whose name is
        # prefix + 'xxxxxx.xxx' in that directory.
        p = tempfile.gettempprefix() + "xxxxxx.xxx"
        d = tempfile.mkdtemp(prefix="")
        try:
            p = os.path.join(d, p)
            try:
                fd = os.open(p, os.O_RDWR | os.O_CREAT)
            except:
                self.failOnException("os.open")
            os.close(fd)
            os.unlink(p)
        finally:
            os.rmdir(d) 
Example 2
Project: fs_image   Author: facebookincubator   File: fs_utils.py    MIT License 6 votes vote down vote up
def open_for_read_decompress(path):
    'Wraps `open(path, "rb")` to add transparent `.zst` or `.gz` decompression.'
    path = Path(path)
    if path.endswith(b'.zst'):
        decompress = 'zstd'
    elif path.endswith(b'.gz') or path.endswith(b'.tgz'):
        decompress = 'gzip'
    else:
        with open(path, 'rb') as f:
            yield f
        return
    with subprocess.Popen([
        decompress, '--decompress', '--stdout', path,
    ], stdout=subprocess.PIPE) as proc:
        yield proc.stdout
    check_popen_returncode(proc) 
Example 3
Project: leapp-repository   Author: oamg   File: lib_backup.py    Apache License 2.0 6 votes vote down vote up
def backup_file(path):
    """ Backup the file specified by path and return the path to the backup file. """
    backup_path = '%s.leapp-backup' % path
    try:
        fd = os.open(backup_path, os.O_CREAT | os.O_EXCL, 0o600)
    except OSError as e:
        if e.errno == errno.EEXIST:
            file_name = os.path.basename(backup_path)
            fd, backup_path = tempfile.mkstemp(prefix=file_name + '.',
                                               dir=os.path.dirname(backup_path))
        else:
            raise
    os.close(fd)

    shutil.copyfile(path, backup_path)
    return backup_path 
Example 4
Project: leapp-repository   Author: oamg   File: ntp2chrony.py    Apache License 2.0 6 votes vote down vote up
def parse_step_tickers(self):
        if not self.step_tickers_path:
            return

        path = os.path.join(self.root_dir, self.step_tickers_path)
        if not os.path.isfile(path):
            logging.info("Missing %s", path)
            return

        with open(path, encoding=self.file_encoding) as f:
            for line in f:
                line = line[:line.find('#')]

                words = line.split()

                if not words:
                    continue

                self.step_tickers.extend(words) 
Example 5
Project: leapp-repository   Author: oamg   File: ntp2chrony.py    Apache License 2.0 6 votes vote down vote up
def parse_ntp_conf(self, path=None):
        if path is None:
            path = os.path.join(self.root_dir, self.ntp_conf_path)

        with open(path, encoding=self.file_encoding) as f:
            logging.info("Reading %s", path)

            for line in f:
                line = line[:line.find('#')]

                words = line.split()

                if not words:
                    continue

                if not self.parse_directive(words):
                    self.ignored_lines.append(line) 
Example 6
Project: leapp-repository   Author: oamg   File: ntp2chrony.py    Apache License 2.0 6 votes vote down vote up
def parse_keys(self, words):
        keyfile = words[0]
        path = os.path.join(self.root_dir, keyfile)
        if not os.path.isfile(path):
            logging.info("Missing %s", path)
            return False

        with open(path, encoding=self.file_encoding) as f:
            logging.info("Reading %s", path)
            keys = []
            for line in f:
                words = line.split()
                if len(words) < 3 or not words[0].isdigit():
                    continue
                keys.append((int(words[0]), words[1], words[2]))

            self.keyfile = keyfile
            self.keys = keys

        return True 
Example 7
Project: pyblish-win   Author: pyblish   File: tempfile.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def _mkstemp_inner(dir, pre, suf, flags):
    """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile."""

    names = _get_candidate_names()

    for seq in xrange(TMP_MAX):
        name = names.next()
        file = _os.path.join(dir, pre + name + suf)
        try:
            fd = _os.open(file, flags, 0600)
            _set_cloexec(fd)
            return (fd, _os.path.abspath(file))
        except OSError, e:
            if e.errno == _errno.EEXIST:
                continue # try again
            if _os.name == 'nt' and e.errno == _errno.EACCES:
                # On windows, when a directory with the chosen name already
                # exists, EACCES error code is returned instead of EEXIST.
                continue
            raise 
Example 8
Project: pyblish-win   Author: pyblish   File: pty.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def slave_open(tty_name):
    """slave_open(tty_name) -> slave_fd
    Open the pty slave and acquire the controlling terminal, returning
    opened filedescriptor.
    Deprecated, use openpty() instead."""

    result = os.open(tty_name, os.O_RDWR)
    try:
        from fcntl import ioctl, I_PUSH
    except ImportError:
        return result
    try:
        ioctl(result, I_PUSH, "ptem")
        ioctl(result, I_PUSH, "ldterm")
    except IOError:
        pass
    return result 
Example 9
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def get_message(self, key):
        """Return a Message representation or raise a KeyError."""
        subpath = self._lookup(key)
        f = open(os.path.join(self._path, subpath), 'r')
        try:
            if self._factory:
                msg = self._factory(f)
            else:
                msg = MaildirMessage(f)
        finally:
            f.close()
        subdir, name = os.path.split(subpath)
        msg.set_subdir(subdir)
        if self.colon in name:
            msg.set_info(name.split(self.colon)[-1])
        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
        return msg 
Example 10
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def _create_tmp(self):
        """Create a file in the tmp subdirectory and open and return it."""
        now = time.time()
        hostname = socket.gethostname()
        if '/' in hostname:
            hostname = hostname.replace('/', r'\057')
        if ':' in hostname:
            hostname = hostname.replace(':', r'\072')
        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
                                    Maildir._count, hostname)
        path = os.path.join(self._path, 'tmp', uniq)
        try:
            os.stat(path)
        except OSError, e:
            if e.errno == errno.ENOENT:
                Maildir._count += 1
                try:
                    return _create_carefully(path)
                except OSError, e:
                    if e.errno != errno.EEXIST:
                        raise 
Example 11
Project: pyblish-win   Author: pyblish   File: test_gzip.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_io_on_closed_object(self):
        # Test that I/O operations on closed GzipFile objects raise a
        # ValueError, just like the corresponding functions on file objects.

        # Write to a file, open it for reading, then close it.
        self.test_write()
        f = gzip.GzipFile(self.filename, 'r')
        f.close()
        with self.assertRaises(ValueError):
            f.read(1)
        with self.assertRaises(ValueError):
            f.seek(0)
        with self.assertRaises(ValueError):
            f.tell()
        # Open the file for writing, then close it.
        f = gzip.GzipFile(self.filename, 'w')
        f.close()
        with self.assertRaises(ValueError):
            f.write('')
        with self.assertRaises(ValueError):
            f.flush() 
Example 12
Project: pyblish-win   Author: pyblish   File: test_gzip.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_many_append(self):
        # Bug #1074261 was triggered when reading a file that contained
        # many, many members.  Create such a file and verify that reading it
        # works.
        with gzip.open(self.filename, 'wb', 9) as f:
            f.write('a')
        for i in range(0, 200):
            with gzip.open(self.filename, "ab", 9) as f: # append
                f.write('a')

        # Try reading the file
        with gzip.open(self.filename, "rb") as zgfile:
            contents = ""
            while 1:
                ztxt = zgfile.read(8192)
                contents += ztxt
                if not ztxt: break
        self.assertEqual(contents, 'a'*201) 
Example 13
Project: pyblish-win   Author: pyblish   File: test_fcntl.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_fcntl_64_bit(self):
        # Issue #1309352: fcntl shouldn't fail when the third arg fits in a
        # C 'long' but not in a C 'int'.
        try:
            cmd = fcntl.F_NOTIFY
            # This flag is larger than 2**31 in 64-bit builds
            flags = fcntl.DN_MULTISHOT
        except AttributeError:
            self.skipTest("F_NOTIFY or DN_MULTISHOT unavailable")
        fd = os.open(os.path.dirname(os.path.abspath(TESTFN)), os.O_RDONLY)
        try:
            # This will raise OverflowError if issue1309352 is present.
            fcntl.fcntl(fd, cmd, flags)
        except IOError:
            pass  # Running on a system that doesn't support these flags.
        finally:
            os.close(fd) 
Example 14
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_closerange(self):
        first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
        # We must allocate two consecutive file descriptors, otherwise
        # it will mess up other file descriptors (perhaps even the three
        # standard ones).
        second = os.dup(first)
        try:
            retries = 0
            while second != first + 1:
                os.close(first)
                retries += 1
                if retries > 10:
                    # XXX test skipped
                    self.skipTest("couldn't allocate two consecutive fds")
                first, second = second, os.dup(second)
        finally:
            os.close(second)
        # close a fd that is open, and one that isn't
        os.closerange(first, first + 2)
        self.assertRaises(OSError, os.write, first, "a") 
Example 15
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_urandom_failure(self):
        # Check urandom() failing when it is not able to open /dev/random.
        # We spawn a new process to make the test more robust (if getrlimit()
        # failed to restore the file descriptor limit after this, the whole
        # test suite would crash; this actually happened on the OS X Tiger
        # buildbot).
        code = """if 1:
            import errno
            import os
            import resource

            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit))
            try:
                os.urandom(16)
            except OSError as e:
                assert e.errno == errno.EMFILE, e.errno
            else:
                raise AssertionError("OSError not raised")
            """
        assert_python_ok('-c', code) 
Example 16
Project: fs_image   Author: facebookincubator   File: unshare.py    MIT License 5 votes vote down vote up
def __init__(self, namespaces: Iterable[Namespace]):
        self._namespaces = frozenset(namespaces)
        self._keepalive_proc = None
        # Instead of using `nsenter --target KEEPALIVE_PID`, we will
        # actually open all the namespace FDs in the current process.  This
        # eliminates an important failure mode at runtime.
        self._namespace_to_file = None
        self._root_fd = None 
Example 17
Project: fs_image   Author: facebookincubator   File: unshare.py    MIT License 5 votes vote down vote up
def __exit__(self, exc_type, exc_val, exc_tb):
        if self._namespace_to_file is not None:
            # We need to close the files to let the namespaces be destroyed
            for f in self._namespace_to_file.values():
                try:
                    f.close()
                # Covering this realistically seems really hard, since this
                # should never fail. Manual test:
                #
                #   >>> log = logging.getLogger('moo')
                #   >>> f = open('/proc/self/ns/mnt', 'rb')
                #   >>> try:
                #   ...     1/0
                #   ... except:
                #   ...     log.exception(f'Closing namespace file {f.name}')
                #   ...
                #   Closing namespace file /proc/self/ns/mnt
                #   Traceback (most recent call last):
                #     File "<stdin>", line 2, in <module>
                #     ZeroDivisionError: division by zero
                except BaseException:  # pragma: no cover
                    log.exception(f'Closing namespace file {f.name}')
            self._namespace_to_file = None

        try:
            os.close(self._root_fd)
        # Same coverage story as above for the `f.close()`
        except BaseException:  # pragma: no cover
            log.exception(f'Closing root directory FD {self._root_fd}')
        self._root_fd = None

        if self._keepalive_proc:
            try:
                self._keepalive_proc.stdin.close()  # "Normally" won't fail
                if self._keepalive_proc.wait() != 0:  # prag
                    log.warning('Unshare keepalive exited with {}'.format(
                        self._keepalive_proc.returncode
                    ))
            finally:
                self._keepalive_proc = None
            # By this point, the namespaces should be getting torn down. 
Example 18
Project: fs_image   Author: facebookincubator   File: common.py    MIT License 5 votes vote down vote up
def open_fd(path: AnyStr, flags) -> int:
    # If you ever need **NOT** to set one of these very sane defaults, add a
    # clearly named keyword-only arg.
    fd = os.open(path, flags=flags | os.O_NOCTTY | os.O_CLOEXEC)
    try:
        yield fd
    finally:
        os.close(fd) 
Example 19
Project: fs_image   Author: facebookincubator   File: test_subvolume_garbage_collector.py    MIT License 5 votes vote down vote up
def _touch(self, *path):
        with open(os.path.join(*path), 'a'):
            pass 
Example 20
Project: fs_image   Author: facebookincubator   File: test_subvolume_garbage_collector.py    MIT License 5 votes vote down vote up
def test_no_gc_due_to_lock(self):
        with self._gc_test_case() as n:
            fd = os.open(n.subs_dir, os.O_RDONLY)
            try:
                fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
                self._gc_only(n)

                # Sneak in a test that new subvolume creation fails when
                # its refcount already exists.
                with tempfile.TemporaryDirectory() as json_dir, \
                     self.assertRaisesRegex(
                         RuntimeError, 'Refcount already exists:',
                     ):
                    sgc.subvolume_garbage_collector([
                        '--refcounts-dir', n.refs_dir,
                        '--subvolumes-dir', n.subs_dir,
                        # This refcount was created by `_gc_test_case`.
                        '--new-subvolume-wrapper-dir', '3link:1',
                        '--new-subvolume-json', os.path.join(json_dir, 'OUT'),
                    ])

            finally:
                    os.close(fd)

            self.assertEqual(
                n.kept_refs | n.gcd_refs, set(os.listdir(n.refs_dir))
            )
            self.assertEqual(
                n.kept_subs | n.gcd_subs, set(os.listdir(n.subs_dir))
            ) 
Example 21
Project: fs_image   Author: facebookincubator   File: fs_utils.py    MIT License 5 votes vote down vote up
def read_text(self) -> str:
        with open(self) as infile:
            return infile.read() 
Example 22
Project: fs_image   Author: facebookincubator   File: fs_utils.py    MIT License 5 votes vote down vote up
def create_ro(path, mode):
    '`open` that creates (and never overwrites) a file with mode `a+r`.'
    def ro_opener(path, flags):
        return os.open(
            path,
            (flags & ~os.O_TRUNC) | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC,
            mode=stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH,
        )
    return open(path, mode, opener=ro_opener) 
Example 23
Project: leapp-repository   Author: oamg   File: ntp2chrony.py    Apache License 2.0 5 votes vote down vote up
def write_file(self, path, mode, content, backup):
        path = self.root_dir + path
        if backup and os.path.isfile(path):
            os.rename(path, path + ".old")

        with open(os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL, mode), "w",
                  encoding=self.file_encoding) as f:
            logging.info("Writing %s", path)
            f.write(u"" + content)

        # Fix SELinux context if restorecon is installed
        try:
            subprocess.call(["restorecon", path])
        except OSError:
            pass 
Example 24
Project: clikit   Author: sdispater   File: terminal.py    MIT License 5 votes vote down vote up
def _get_terminal_size_linux(self):
        def ioctl_GWINSZ(fd):
            try:
                import fcntl
                import termios

                cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
                return cr
            except:
                pass

        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
        if not cr:
            try:
                fd = os.open(os.ctermid(), os.O_RDONLY)
                cr = ioctl_GWINSZ(fd)
                os.close(fd)
            except:
                pass

        if not cr:
            try:
                cr = (os.environ["LINES"], os.environ["COLUMNS"])
            except:
                return None

        return int(cr[1]), int(cr[0]) 
Example 25
Project: pyblish-win   Author: pyblish   File: tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _stat(fn):
        try:
            f = open(fn)
        except IOError:
            raise _os.error
        f.close() 
Example 26
Project: pyblish-win   Author: pyblish   File: tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _get_default_tempdir():
    """Calculate the default directory to use for temporary files.
    This routine should be called exactly once.

    We determine whether or not a candidate temp dir is usable by
    trying to create and write to a file in that directory.  If this
    is successful, the test file is deleted.  To prevent denial of
    service, the name of the test file must be randomized."""

    namer = _RandomNameSequence()
    dirlist = _candidate_tempdir_list()
    flags = _text_openflags

    for dir in dirlist:
        if dir != _os.curdir:
            dir = _os.path.normcase(_os.path.abspath(dir))
        # Try only a few names per directory.
        for seq in xrange(100):
            name = namer.next()
            filename = _os.path.join(dir, name)
            try:
                fd = _os.open(filename, flags, 0o600)
                try:
                    try:
                        with _io.open(fd, 'wb', closefd=False) as fp:
                            fp.write(b'blat')
                    finally:
                        _os.close(fd)
                finally:
                    _os.unlink(filename)
                return dir
            except (OSError, IOError) as e:
                if e.args[0] != _errno.EEXIST:
                    break # no point trying more names in this directory
                pass
    raise IOError, (_errno.ENOENT,
                    ("No usable temporary directory found in %s" % dirlist)) 
Example 27
Project: pyblish-win   Author: pyblish   File: tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def mkstemp(suffix="", prefix=template, dir=None, text=False):
    """User-callable function to create and return a unique temporary
    file.  The return value is a pair (fd, name) where fd is the
    file descriptor returned by os.open, and name is the filename.

    If 'suffix' is specified, the file name will end with that suffix,
    otherwise there will be no suffix.

    If 'prefix' is specified, the file name will begin with that prefix,
    otherwise a default prefix is used.

    If 'dir' is specified, the file will be created in that directory,
    otherwise a default directory is used.

    If 'text' is specified and true, the file is opened in text
    mode.  Else (the default) the file is opened in binary mode.  On
    some operating systems, this makes no difference.

    The file is readable and writable only by the creating user ID.
    If the operating system uses permission bits to indicate whether a
    file is executable, the file is executable by no one. The file
    descriptor is not inherited by children of this process.

    Caller is responsible for deleting the file when done with it.
    """

    if dir is None:
        dir = gettempdir()

    if text:
        flags = _text_openflags
    else:
        flags = _bin_openflags

    return _mkstemp_inner(dir, prefix, suffix, flags) 
Example 28
Project: pyblish-win   Author: pyblish   File: tarfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _check(self, mode=None):
        """Check if TarFile is still open, and if the operation's mode
           corresponds to TarFile's mode.
        """
        if self.closed:
            raise IOError("%s is closed" % self.__class__.__name__)
        if mode is not None and self.mode not in mode:
            raise IOError("bad operation for mode %r" % self.mode) 
Example 29
Project: pyblish-win   Author: pyblish   File: tarfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def is_tarfile(name):
    """Return True if name points to a tar archive that we
       are able to handle, else return False.
    """
    try:
        t = open(name)
        t.close()
        return True
    except TarError:
        return False 
Example 30
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_string(self, key):
        """Return a string representation or raise a KeyError."""
        f = open(os.path.join(self._path, self._lookup(key)), 'r')
        try:
            return f.read()
        finally:
            f.close() 
Example 31
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_file(self, key):
        """Return a file-like representation or raise a KeyError."""
        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
        return _ProxyFile(f) 
Example 32
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, path, factory=None, create=True):
        """Initialize a single-file mailbox."""
        Mailbox.__init__(self, path, factory, create)
        try:
            f = open(self._path, 'rb+')
        except IOError, e:
            if e.errno == errno.ENOENT:
                if create:
                    f = open(self._path, 'wb+')
                else:
                    raise NoSuchMailboxError(self._path)
            elif e.errno in (errno.EACCES, errno.EROFS):
                f = open(self._path, 'rb')
            else:
                raise 
Example 33
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, path, factory=None, create=True):
        """Initialize an MH instance."""
        Mailbox.__init__(self, path, factory, create)
        if not os.path.exists(self._path):
            if create:
                os.mkdir(self._path, 0700)
                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
            else:
                raise NoSuchMailboxError(self._path)
        self._locked = False 
Example 34
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def remove(self, key):
        """Remove the keyed message; raise KeyError if it doesn't exist."""
        path = os.path.join(self._path, str(key))
        try:
            f = open(path, 'rb+')
        except IOError, e:
            if e.errno == errno.ENOENT:
                raise KeyError('No message with key: %s' % key)
            else:
                raise 
Example 35
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __setitem__(self, key, message):
        """Replace the keyed message; raise KeyError if it doesn't exist."""
        path = os.path.join(self._path, str(key))
        try:
            f = open(path, 'rb+')
        except IOError, e:
            if e.errno == errno.ENOENT:
                raise KeyError('No message with key: %s' % key)
            else:
                raise 
Example 36
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_string(self, key):
        """Return a string representation or raise a KeyError."""
        try:
            if self._locked:
                f = open(os.path.join(self._path, str(key)), 'r+')
            else:
                f = open(os.path.join(self._path, str(key)), 'r')
        except IOError, e:
            if e.errno == errno.ENOENT:
                raise KeyError('No message with key: %s' % key)
            else:
                raise 
Example 37
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_file(self, key):
        """Return a file-like representation or raise a KeyError."""
        try:
            f = open(os.path.join(self._path, str(key)), 'rb')
        except IOError, e:
            if e.errno == errno.ENOENT:
                raise KeyError('No message with key: %s' % key)
            else:
                raise 
Example 38
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def lock(self):
        """Lock the mailbox."""
        if not self._locked:
            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
            _lock_file(self._file)
            self._locked = True 
Example 39
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_sequences(self):
        """Return a name-to-key-list dictionary to define each sequence."""
        results = {}
        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
        try:
            all_keys = set(self.keys())
            for line in f:
                try:
                    name, contents = line.split(':')
                    keys = set()
                    for spec in contents.split():
                        if spec.isdigit():
                            keys.add(int(spec))
                        else:
                            start, stop = (int(x) for x in spec.split('-'))
                            keys.update(range(start, stop + 1))
                    results[name] = [key for key in sorted(keys) \
                                         if key in all_keys]
                    if len(results[name]) == 0:
                        del results[name]
                except ValueError:
                    raise FormatError('Invalid sequence specification: %s' %
                                      line.rstrip())
        finally:
            f.close()
        return results 
Example 40
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def set_sequences(self, sequences):
        """Set sequences using the given name-to-key-list dictionary."""
        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
        try:
            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
            for name, keys in sequences.iteritems():
                if len(keys) == 0:
                    continue
                f.write('%s:' % name)
                prev = None
                completing = False
                for key in sorted(set(keys)):
                    if key - 1 == prev:
                        if not completing:
                            completing = True
                            f.write('-')
                    elif completing:
                        completing = False
                        f.write('%s %s' % (prev, key))
                    else:
                        f.write(' %s' % key)
                    prev = key
                if completing:
                    f.write(str(prev) + '\n')
                else:
                    f.write('\n')
        finally:
            _sync_close(f) 
Example 41
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _create_temporary(path):
    """Create a temp file based on path and open for reading and writing."""
    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
                                              socket.gethostname(),
                                              os.getpid())) 
Example 42
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def next(self):
        if not self.boxes:
            return None
        fn = self.boxes.pop()
        fp = open(os.path.join(self.dirname, fn))
        msg = self.factory(fp)
        try:
            msg._mh_msgno = fn
        except (AttributeError, TypeError):
            pass
        return msg 
Example 43
Project: pyblish-win   Author: pyblish   File: getpass.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def unix_getpass(prompt='Password: ', stream=None):
    """Prompt for a password, with echo turned off.

    Args:
      prompt: Written on stream to ask for the input.  Default: 'Password: '
      stream: A writable file object to display the prompt.  Defaults to
              the tty.  If no tty is available defaults to sys.stderr.
    Returns:
      The seKr3t input.
    Raises:
      EOFError: If our input tty or stdin was closed.
      GetPassWarning: When we were unable to turn echo off on the input.

    Always restores terminal settings before returning.
    """
    fd = None
    tty = None
    try:
        # Always try reading and writing directly on the tty first.
        fd = os.open('/dev/tty', os.O_RDWR|os.O_NOCTTY)
        tty = os.fdopen(fd, 'w+', 1)
        input = tty
        if not stream:
            stream = tty
    except EnvironmentError, e:
        # If that fails, see if stdin can be controlled.
        try:
            fd = sys.stdin.fileno()
        except (AttributeError, ValueError):
            passwd = fallback_getpass(prompt, stream)
        input = sys.stdin
        if not stream:
            stream = sys.stderr 
Example 44
Project: pyblish-win   Author: pyblish   File: test_zipimport.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def doTest(self, expected_ext, files, *modules, **kw):
        z = ZipFile(TEMP_ZIP, "w")
        try:
            for name, (mtime, data) in files.items():
                zinfo = ZipInfo(name, time.localtime(mtime))
                zinfo.compress_type = self.compression
                z.writestr(zinfo, data)
            z.close()

            stuff = kw.get("stuff", None)
            if stuff is not None:
                # Prepend 'stuff' to the start of the zipfile
                f = open(TEMP_ZIP, "rb")
                data = f.read()
                f.close()

                f = open(TEMP_ZIP, "wb")
                f.write(stuff)
                f.write(data)
                f.close()

            sys.path.insert(0, TEMP_ZIP)

            mod = __import__(".".join(modules), globals(), locals(),
                             ["__dummy__"])

            call = kw.get('call')
            if call is not None:
                call(mod)

            if expected_ext:
                file = mod.get_file()
                self.assertEqual(file, os.path.join(TEMP_ZIP,
                                 *modules) + expected_ext)
        finally:
            z.close()
            os.remove(TEMP_ZIP) 
Example 45
Project: pyblish-win   Author: pyblish   File: test_zipimport.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def testEmptyFile(self):
        test_support.unlink(TESTMOD)
        open(TESTMOD, 'w+').close()
        self.assertZipFailure(TESTMOD) 
Example 46
Project: pyblish-win   Author: pyblish   File: test_zipimport.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def testNotZipFile(self):
        test_support.unlink(TESTMOD)
        fp = open(TESTMOD, 'w+')
        fp.write('a' * 22)
        fp.close()
        self.assertZipFailure(TESTMOD)

    # XXX: disabled until this works on Big-endian machines 
Example 47
Project: pyblish-win   Author: pyblish   File: test_zipimport.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _testBogusZipFile(self):
        test_support.unlink(TESTMOD)
        fp = open(TESTMOD, 'w+')
        fp.write(struct.pack('=I', 0x06054B50))
        fp.write('a' * 18)
        fp.close()
        z = zipimport.zipimporter(TESTMOD)

        try:
            self.assertRaises(TypeError, z.find_module, None)
            self.assertRaises(TypeError, z.load_module, None)
            self.assertRaises(TypeError, z.is_package, None)
            self.assertRaises(TypeError, z.get_code, None)
            self.assertRaises(TypeError, z.get_data, None)
            self.assertRaises(TypeError, z.get_source, None)

            error = zipimport.ZipImportError
            self.assertEqual(z.find_module('abc'), None)

            self.assertRaises(error, z.load_module, 'abc')
            self.assertRaises(error, z.get_code, 'abc')
            self.assertRaises(IOError, z.get_data, 'abc')
            self.assertRaises(error, z.get_source, 'abc')
            self.assertRaises(error, z.is_package, 'abc')
        finally:
            zipimport._zip_directory_cache.clear() 
Example 48
Project: pyblish-win   Author: pyblish   File: test_gzip.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_unicode_filename(self):
        unicode_filename = test_support.TESTFN_UNICODE
        try:
            unicode_filename.encode(test_support.TESTFN_ENCODING)
        except (UnicodeError, TypeError):
            self.skipTest("Requires unicode filenames support")
        with gzip.GzipFile(unicode_filename, "wb") as f:
            f.write(data1 * 50)
        with gzip.GzipFile(unicode_filename, "rb") as f:
            self.assertEqual(f.read(), data1 * 50)
        # Sanity check that we are actually operating on the right file.
        with open(unicode_filename, 'rb') as fobj, \
             gzip.GzipFile(fileobj=fobj, mode="rb") as f:
            self.assertEqual(f.read(), data1 * 50) 
Example 49
Project: pyblish-win   Author: pyblish   File: test_gzip.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_zero_padded_file(self):
        with gzip.GzipFile(self.filename, "wb") as f:
            f.write(data1 * 50)

        # Pad the file with zeroes
        with open(self.filename, "ab") as f:
            f.write("\x00" * 50)

        with gzip.GzipFile(self.filename, "rb") as f:
            d = f.read()
            self.assertEqual(d, data1 * 50, "Incorrect data in file") 
Example 50
Project: pyblish-win   Author: pyblish   File: test_gzip.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_fileobj_from_fdopen(self):
        # Issue #13781: Creating a GzipFile using a fileobj from os.fdopen()
        # should not embed the fake filename "<fdopen>" in the output file.
        fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT)
        with os.fdopen(fd, "wb") as f:
            with gzip.GzipFile(fileobj=f, mode="w") as g:
                self.assertEqual(g.name, "") 
Example 51
Project: pyblish-win   Author: pyblish   File: test_fcntl.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_fcntl_fileno(self):
        # the example from the library docs
        self.f = open(TESTFN, 'w')
        rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
        if verbose:
            print 'Status from fcntl with O_NONBLOCK: ', rv
        if sys.platform not in ['os2emx']:
            rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
            if verbose:
                print 'String from fcntl with F_SETLKW: ', repr(rv)
        self.f.close() 
Example 52
Project: pyblish-win   Author: pyblish   File: test_fcntl.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_fcntl_file_descriptor(self):
        # again, but pass the file rather than numeric descriptor
        self.f = open(TESTFN, 'w')
        rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK)
        if sys.platform not in ['os2emx']:
            rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
        self.f.close() 
Example 53
Project: pyblish-win   Author: pyblish   File: test_tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, dir, pre, suf):
            self.name = tempfile.mktemp(dir=dir, prefix=pre, suffix=suf)
            # Create the file.  This will raise an exception if it's
            # mysteriously appeared in the meanwhile.
            os.close(os.open(self.name, self._bflags, 0600)) 
Example 54
Project: pyblish-win   Author: pyblish   File: test_asyncore.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_recv(self):
        fd = os.open(TESTFN, os.O_RDONLY)
        w = asyncore.file_wrapper(fd)
        os.close(fd)

        self.assertNotEqual(w.fd, fd)
        self.assertNotEqual(w.fileno(), fd)
        self.assertEqual(w.recv(13), "It's not dead")
        self.assertEqual(w.read(6), ", it's")
        w.close()
        self.assertRaises(OSError, w.read, 1) 
Example 55
Project: pyblish-win   Author: pyblish   File: test_asyncore.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_send(self):
        d1 = "Come again?"
        d2 = "I want to buy some cheese."
        fd = os.open(TESTFN, os.O_WRONLY | os.O_APPEND)
        w = asyncore.file_wrapper(fd)
        os.close(fd)

        w.write(d1)
        w.send(d2)
        w.close()
        self.assertEqual(file(TESTFN).read(), self.d + d1 + d2) 
Example 56
Project: pyblish-win   Author: pyblish   File: test_asyncore.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_dispatcher(self):
        fd = os.open(TESTFN, os.O_RDONLY)
        data = []
        class FileDispatcher(asyncore.file_dispatcher):
            def handle_read(self):
                data.append(self.recv(29))
        s = FileDispatcher(fd)
        os.close(fd)
        asyncore.loop(timeout=0.01, use_poll=True, count=2)
        self.assertEqual(b"".join(data), self.d) 
Example 57
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stdin_filedes(self):
        # stdin is set to open file descriptor
        tf = tempfile.TemporaryFile()
        d = tf.fileno()
        os.write(d, "pear")
        os.lseek(d, 0, 0)
        p = subprocess.Popen([sys.executable, "-c",
                         'import sys; sys.exit(sys.stdin.read() == "pear")'],
                         stdin=d)
        p.wait()
        self.assertEqual(p.returncode, 1) 
Example 58
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stdin_fileobj(self):
        # stdin is set to open file object
        tf = tempfile.TemporaryFile()
        tf.write("pear")
        tf.seek(0)
        p = subprocess.Popen([sys.executable, "-c",
                         'import sys; sys.exit(sys.stdin.read() == "pear")'],
                         stdin=tf)
        p.wait()
        self.assertEqual(p.returncode, 1) 
Example 59
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stdout_filedes(self):
        # stdout is set to open file descriptor
        tf = tempfile.TemporaryFile()
        d = tf.fileno()
        p = subprocess.Popen([sys.executable, "-c",
                          'import sys; sys.stdout.write("orange")'],
                         stdout=d)
        p.wait()
        os.lseek(d, 0, 0)
        self.assertEqual(os.read(d, 1024), "orange") 
Example 60
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stdout_fileobj(self):
        # stdout is set to open file object
        tf = tempfile.TemporaryFile()
        p = subprocess.Popen([sys.executable, "-c",
                          'import sys; sys.stdout.write("orange")'],
                         stdout=tf)
        p.wait()
        tf.seek(0)
        self.assertEqual(tf.read(), "orange") 
Example 61
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stderr_filedes(self):
        # stderr is set to open file descriptor
        tf = tempfile.TemporaryFile()
        d = tf.fileno()
        p = subprocess.Popen([sys.executable, "-c",
                          'import sys; sys.stderr.write("strawberry")'],
                         stderr=d)
        p.wait()
        os.lseek(d, 0, 0)
        self.assertStderrEqual(os.read(d, 1024), "strawberry") 
Example 62
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_stdout_stderr_file(self):
        # capture stdout and stderr to the same open file
        tf = tempfile.TemporaryFile()
        p = subprocess.Popen([sys.executable, "-c",
                          'import sys;'
                          'sys.stdout.write("apple");'
                          'sys.stdout.flush();'
                          'sys.stderr.write("orange")'],
                         stdout=tf,
                         stderr=tf)
        p.wait()
        tf.seek(0)
        self.assertStderrEqual(tf.read(), "appleorange") 
Example 63
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_no_leaking(self):
        # Make sure we leak no resources
        if not mswindows:
            max_handles = 1026 # too much for most UNIX systems
        else:
            max_handles = 2050 # too much for (at least some) Windows setups
        handles = []
        try:
            for i in range(max_handles):
                try:
                    handles.append(os.open(test_support.TESTFN,
                                           os.O_WRONLY | os.O_CREAT))
                except OSError as e:
                    if e.errno != errno.EMFILE:
                        raise
                    break
            else:
                self.skipTest("failed to reach the file descriptor limit "
                    "(tried %d)" % max_handles)
            # Close a couple of them (should be enough for a subprocess)
            for i in range(10):
                os.close(handles.pop())
            # Loop creating some subprocesses. If one of them leaks some fds,
            # the next loop iteration will fail by reaching the max fd limit.
            for i in range(15):
                p = subprocess.Popen([sys.executable, "-c",
                                      "import sys;"
                                      "sys.stdout.write(sys.stdin.read())"],
                                     stdin=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
                data = p.communicate(b"lime")[0]
                self.assertEqual(data, b"lime")
        finally:
            for h in handles:
                os.close(h)
            test_support.unlink(test_support.TESTFN) 
Example 64
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_leaking_fds_on_error(self):
        # see bug #5179: Popen leaks file descriptors to PIPEs if
        # the child fails to execute; this will eventually exhaust
        # the maximum number of open fds. 1024 seems a very common
        # value for that limit, but Windows has 2048, so we loop
        # 1024 times (each call leaked two fds).
        for i in range(1024):
            # Windows raises IOError.  Others raise OSError.
            with self.assertRaises(EnvironmentError) as c:
                subprocess.Popen(['nonexisting_i_hope'],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # ignore errors that indicate the command was not found
            if c.exception.errno not in (errno.ENOENT, errno.EACCES):
                raise c.exception 
Example 65
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _execute_child(
                self, args, executable, preexec_fn, close_fds, cwd, env,
                universal_newlines, startupinfo, creationflags, shell, to_close,
                p2cread, p2cwrite,
                c2pread, c2pwrite,
                errread, errwrite):
            try:
                subprocess.Popen._execute_child(
                        self, args, executable, preexec_fn, close_fds,
                        cwd, env, universal_newlines,
                        startupinfo, creationflags, shell, to_close,
                        p2cread, p2cwrite,
                        c2pread, c2pwrite,
                        errread, errwrite)
            finally:
                # Open a bunch of file descriptors and verify that
                # none of them are the same as the ones the Popen
                # instance is using for stdin/stdout/stderr.
                devzero_fds = [os.open("/dev/zero", os.O_RDONLY)
                               for _ in range(8)]
                try:
                    for fd in devzero_fds:
                        self._testcase.assertNotIn(
                                fd, (p2cwrite, c2pread, errread))
                finally:
                    for fd in devzero_fds:
                        os.close(fd) 
Example 66
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def check_tempfile(self, name):
        # make sure it doesn't already exist:
        self.assertFalse(os.path.exists(name),
                    "file already exists for temporary file")
        # make sure we can create the file
        open(name, "w")
        self.files.append(name) 
Example 67
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_tmpfile(self):
        # As with test_tmpnam() below, the Windows implementation of tmpfile()
        # attempts to create a file in the root directory of the current drive.
        # On Vista and Server 2008, this test will always fail for normal users
        # as writing to the root directory requires elevated privileges.  With
        # XP and below, the semantics of tmpfile() are the same, but the user
        # running the test is more likely to have administrative privileges on
        # their account already.  If that's the case, then os.tmpfile() should
        # work.  In order to make this test as useful as possible, rather than
        # trying to detect Windows versions or whether or not the user has the
        # right permissions, just try and create a file in the root directory
        # and see if it raises a 'Permission denied' OSError.  If it does, then
        # test that a subsequent call to os.tmpfile() raises the same error. If
        # it doesn't, assume we're on XP or below and the user running the test
        # has administrative privileges, and proceed with the test as normal.
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", "tmpfile", DeprecationWarning)

            if sys.platform == 'win32':
                name = '\\python_test_os_test_tmpfile.txt'
                if os.path.exists(name):
                    os.remove(name)
                try:
                    fp = open(name, 'w')
                except IOError, first:
                    # open() failed, assert tmpfile() fails in the same way.
                    # Although open() raises an IOError and os.tmpfile() raises an
                    # OSError(), 'args' will be (13, 'Permission denied') in both
                    # cases.
                    try:
                        fp = os.tmpfile()
                    except OSError, second:
                        self.assertEqual(first.args, second.args)
                    else:
                        self.fail("expected os.tmpfile() to raise OSError")
                    return
                else: 
Example 68
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_tmpnam(self):
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning,
                                    r"test_os$")
            warnings.filterwarnings("ignore", "tmpnam", DeprecationWarning)

            name = os.tmpnam()
            if sys.platform in ("win32",):
                # The Windows tmpnam() seems useless.  From the MS docs:
                #
                #     The character string that tmpnam creates consists of
                #     the path prefix, defined by the entry P_tmpdir in the
                #     file STDIO.H, followed by a sequence consisting of the
                #     digit characters '0' through '9'; the numerical value
                #     of this string is in the range 1 - 65,535.  Changing the
                #     definitions of L_tmpnam or P_tmpdir in STDIO.H does not
                #     change the operation of tmpnam.
                #
                # The really bizarre part is that, at least under MSVC6,
                # P_tmpdir is "\\".  That is, the path returned refers to
                # the root of the current drive.  That's a terrible place to
                # put temp files, and, depending on privileges, the user
                # may not even be able to open a file in the root directory.
                self.assertFalse(os.path.exists(name),
                            "file already exists for temporary file")
            else:
                self.check_tempfile(name)

# Test attributes on return values from os.*stat* family. 
Example 69
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        os.mkdir(test_support.TESTFN)
        self.fname = os.path.join(test_support.TESTFN, "f1")
        f = open(self.fname, 'wb')
        f.write("ABC")
        f.close() 
Example 70
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_mkdir(self):
        f = open(test_support.TESTFN, "w")
        try:
            self.assertRaises(WindowsError, os.mkdir, test_support.TESTFN)
        finally:
            f.close()
            os.unlink(test_support.TESTFN) 
Example 71
Project: pyblish-win   Author: pyblish   File: test_posix.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        # create empty file
        fp = open(test_support.TESTFN, 'w+')
        fp.close()
        self.teardown_files = [ test_support.TESTFN ] 
Example 72
Project: pyblish-win   Author: pyblish   File: test_posix.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_fstatvfs(self):
        fp = open(test_support.TESTFN)
        try:
            self.assertTrue(posix.fstatvfs(fp.fileno()))
        finally:
            fp.close() 
Example 73
Project: pyblish-win   Author: pyblish   File: test_posix.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_ftruncate(self):
        fp = open(test_support.TESTFN, 'w+')
        try:
            # we need to have some data to truncate
            fp.write('test')
            fp.flush()
            posix.ftruncate(fp.fileno(), 0)
        finally:
            fp.close() 
Example 74
Project: fs_image   Author: facebookincubator   File: subvolume_garbage_collector.py    MIT License 4 votes vote down vote up
def garbage_collect_subvolumes(refcounts_dir, subvolumes_dir):
    # IMPORTANT: We must list subvolumes BEFORE refcounts. The risk is that
    # this runs concurrently with another build, which will create a new
    # refcount & subvolume (in that order).  If we read refcounts first, we
    # might end up winning the race against the other build, and NOT reading
    # the new refcount.  If we then lose the second part of the race, we
    # would find the subvolume that the other process just created, and
    # delete it.
    subvol_wrappers = set(list_subvolume_wrappers(subvolumes_dir))
    subvol_wrapper_to_nlink = dict(list_refcounts(refcounts_dir))

    # Delete subvolumes (& their wrappers) with insufficient refcounts.
    for subvol_wrapper in subvol_wrappers:
        nlink = subvol_wrapper_to_nlink.get(subvol_wrapper, 0)
        if nlink >= 2:
            if nlink > 2:
                # Not sure how this might happen, but it seems non-fatal...
                log.error(f'{nlink} > 2 links to subvolume {subvol_wrapper}')
            continue
        refcount_path = os.path.join(refcounts_dir, f'{subvol_wrapper}.json')
        log.warning(
            f'Deleting {subvol_wrapper} since its refcount has {nlink} links'
        )
        # Start by unlinking the refcount to dramatically decrease the
        # chance of leaving an orphaned refcount file on disk.  The most
        # obvious way to get an orphaned refcount is for this program to
        # abort between the line that creates the refcount link, and the
        # next line that creates the subvolume wrapper.
        #
        # I do not see a great way to completely eliminate orphan refcount
        # files.  One could try to have a separate pass that flocks the
        # refcount file before removing it, and to also flock the refcount
        # file before creating the wrapper directory.  But, since file
        # creation & flock cannot be atomic, this leaves us open to a race
        # where a concurrent GC pass removes the refcount link immediately
        # after it gets created, so that part of the code would have to be
        # willing to repeat the race until it wins.  In all, that extra
        # complexity is far too ugly compared to the slim risk or leaving
        # some unused refcount files on disk.
        if nlink:
            os.unlink(refcount_path)
        wrapper_path = os.path.join(subvolumes_dir, subvol_wrapper)
        wrapper_content = os.listdir(wrapper_path)
        if len(wrapper_content) > 1:
            raise RuntimeError(f'{wrapper_path} must contain only the subvol')
        if len(wrapper_content) == 1:  # Empty wrappers are OK to GC, too.
            subprocess.check_call([
                'sudo', 'btrfs', 'subvolume', 'delete',
                os.path.join(
                    subvolumes_dir,
                    # Subvols are wrapped in a user-owned temporary directory,
                    # following the convention `{rule name}:{version}/{subvol}`.
                    subvol_wrapper,
                    wrapper_content[0],
                ),
            ])
        os.rmdir(wrapper_path) 
Example 75
Project: pyblish-win   Author: pyblish   File: fileinput.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def readline(self):
        try:
            line = self._buffer[self._bufindex]
        except IndexError:
            pass
        else:
            self._bufindex += 1
            self._lineno += 1
            self._filelineno += 1
            return line
        if not self._file:
            if not self._files:
                return ""
            self._filename = self._files[0]
            self._files = self._files[1:]
            self._filelineno = 0
            self._file = None
            self._isstdin = False
            self._backupfilename = 0
            if self._filename == '-':
                self._filename = '<stdin>'
                self._file = sys.stdin
                self._isstdin = True
            else:
                if self._inplace:
                    self._backupfilename = (
                        self._filename + (self._backup or os.extsep+"bak"))
                    try: os.unlink(self._backupfilename)
                    except os.error: pass
                    # The next few lines may raise IOError
                    os.rename(self._filename, self._backupfilename)
                    self._file = open(self._backupfilename, self._mode)
                    try:
                        perm = os.fstat(self._file.fileno()).st_mode
                    except OSError:
                        self._output = open(self._filename, "w")
                    else:
                        fd = os.open(self._filename,
                                     os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
                                     perm)
                        self._output = os.fdopen(fd, "w")
                        try:
                            if hasattr(os, 'chmod'):
                                os.chmod(self._filename, perm)
                        except OSError:
                            pass
                    self._savestdout = sys.stdout
                    sys.stdout = self._output
                else:
                    # This may raise IOError
                    if self._openhook:
                        self._file = self._openhook(self._filename, self._mode)
                    else:
                        self._file = open(self._filename, self._mode)
        self._buffer = self._file.readlines(self._bufsize)
        self._bufindex = 0
        if not self._buffer:
            self.nextfile()
        # Recursive call
        return self.readline() 
Example 76
Project: pyblish-win   Author: pyblish   File: fileinput.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def hook_encoded(encoding):
    import io
    def openhook(filename, mode):
        mode = mode.replace('U', '').replace('b', '') or 'r'
        return io.open(filename, mode, encoding=encoding, newline='')
    return openhook 
Example 77
Project: pyblish-win   Author: pyblish   File: pty.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def fork():
    """fork() -> (pid, master_fd)
    Fork and make the child a session leader with a controlling terminal."""

    try:
        pid, fd = os.forkpty()
    except (AttributeError, OSError):
        pass
    else:
        if pid == CHILD:
            try:
                os.setsid()
            except OSError:
                # os.forkpty() already set us session leader
                pass
        return pid, fd

    master_fd, slave_fd = openpty()
    pid = os.fork()
    if pid == CHILD:
        # Establish a new session.
        os.setsid()
        os.close(master_fd)

        # Slave becomes stdin/stdout/stderr of child.
        os.dup2(slave_fd, STDIN_FILENO)
        os.dup2(slave_fd, STDOUT_FILENO)
        os.dup2(slave_fd, STDERR_FILENO)
        if (slave_fd > STDERR_FILENO):
            os.close (slave_fd)

        # Explicitly open the tty to make it become a controlling tty.
        tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
        os.close(tmp_fd)
    else:
        os.close(slave_fd)

    # Parent and child process.
    return pid, master_fd 
Example 78
Project: pyblish-win   Author: pyblish   File: tarfile.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def __init__(self, name, mode):
        mode = {
            "r": os.O_RDONLY,
            "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
        }[mode]
        if hasattr(os, "O_BINARY"):
            mode |= os.O_BINARY
        self.fd = os.open(name, mode, 0666) 
Example 79
Project: pyblish-win   Author: pyblish   File: tarfile.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
        """Open a tar archive for reading, writing or appending. Return
           an appropriate TarFile class.

           mode:
           'r' or 'r:*' open for reading with transparent compression
           'r:'         open for reading exclusively uncompressed
           'r:gz'       open for reading with gzip compression
           'r:bz2'      open for reading with bzip2 compression
           'a' or 'a:'  open for appending, creating the file if necessary
           'w' or 'w:'  open for writing without compression
           'w:gz'       open for writing with gzip compression
           'w:bz2'      open for writing with bzip2 compression

           'r|*'        open a stream of tar blocks with transparent compression
           'r|'         open an uncompressed stream of tar blocks for reading
           'r|gz'       open a gzip compressed stream of tar blocks
           'r|bz2'      open a bzip2 compressed stream of tar blocks
           'w|'         open an uncompressed stream for writing
           'w|gz'       open a gzip compressed stream for writing
           'w|bz2'      open a bzip2 compressed stream for writing
        """

        if not name and not fileobj:
            raise ValueError("nothing to open")

        if mode in ("r", "r:*"):
            # Find out which *open() is appropriate for opening the file.
            for comptype in cls.OPEN_METH:
                func = getattr(cls, cls.OPEN_METH[comptype])
                if fileobj is not None:
                    saved_pos = fileobj.tell()
                try:
                    return func(name, "r", fileobj, **kwargs)
                except (ReadError, CompressionError), e:
                    if fileobj is not None:
                        fileobj.seek(saved_pos)
                    continue
            raise ReadError("file could not be opened successfully") 
Example 80
Project: pyblish-win   Author: pyblish   File: test_subprocess.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def check_swap_fds(self, stdin_no, stdout_no, stderr_no):
        # open up some temporary files
        temps = [mkstemp() for i in range(3)]
        temp_fds = [fd for fd, fname in temps]
        try:
            # unlink the files -- we won't need to reopen them
            for fd, fname in temps:
                os.unlink(fname)

            # save a copy of the standard file descriptors
            saved_fds = [os.dup(fd) for fd in range(3)]
            try:
                # duplicate the temp files over the standard fd's 0, 1, 2
                for fd, temp_fd in enumerate(temp_fds):
                    os.dup2(temp_fd, fd)

                # write some data to what will become stdin, and rewind
                os.write(stdin_no, b"STDIN")
                os.lseek(stdin_no, 0, 0)

                # now use those files in the given order, so that subprocess
                # has to rearrange them in the child
                p = subprocess.Popen([sys.executable, "-c",
                    'import sys; got = sys.stdin.read();'
                    'sys.stdout.write("got %s"%got); sys.stderr.write("err")'],
                    stdin=stdin_no,
                    stdout=stdout_no,
                    stderr=stderr_no)
                p.wait()

                for fd in temp_fds:
                    os.lseek(fd, 0, 0)

                out = os.read(stdout_no, 1024)
                err = test_support.strip_python_stderr(os.read(stderr_no, 1024))
            finally:
                for std, saved in enumerate(saved_fds):
                    os.dup2(saved, std)
                    os.close(saved)

            self.assertEqual(out, b"got STDIN")
            self.assertEqual(err, b"err")

        finally:
            for fd in temp_fds:
                os.close(fd)

    # When duping fds, if there arises a situation where one of the fds is
    # either 0, 1 or 2, it is possible that it is overwritten (#12607).
    # This tests all combinations of this.