Python os.html() Examples

def system():
    import platform
    # On some Windows installation (Python 2.4) platform.system() is
    # broken and incorrectly returns 'Microsoft' instead of 'Windows'.
    # http://mail.python.org/pipermail/patches/2007-June/022947.html
    syst = platform.system()
    if syst == 'Microsoft':
        return 'Windows'
    return syst


# Set and get environment variables does not handle unicode strings correctly
# on Windows.

# Acting on os.environ instead of using getenv()/setenv()/unsetenv(),
# as suggested in <http://docs.python.org/library/os.html#os.environ>:
# "Calling putenv() directly does not change os.environ, so it's
# better to modify os.environ." (Same for unsetenv.) 

def run(command):
    # TODO: replace this with fork()
    #       (https://docs.python.org/2/library/os.html#os.fork)
    pid = 0
    if pid == 0:
        # This is the child, we'll try to do some containment here
        try:
            contain(command)
        except Exception:
            traceback.print_exc()
            os._exit(1)  # something went wrong in contain()

    # This is the parent, pid contains the PID of the forked process
    # wait for the forked child and fetch the exit status
    _, status = os.waitpid(pid, 0)
    print('{} exited with status {}'.format(pid, status)) 

def rename(src, dst):
  """os.rename(src, dst) wrapper with support for long paths on Windows.

  Availability: Unix, Windows."""
  if isWindows():
    # On Windows, rename fails if destination exists, see
    # https://docs.python.org/2/library/os.html#os.rename
    try:
      os.rename(_makelongpath(src), _makelongpath(dst))
    except OSError as e:
      if e.errno == errno.EEXIST:
        os.remove(_makelongpath(dst))
        os.rename(_makelongpath(src), _makelongpath(dst))
      else:
        raise
  else:
    os.rename(src, dst) 

def fsync(self):
        if self._oplog:
            # https://docs.python.org/2/library/os.html#os.fsync
            self._oplog.flush()
            self._last_flush_time  = time()
            self._writes_unflushed = 0
            return os.fsync(self._oplog.fileno()) 

def add(self, package=None, filename=None, glob=None):
        """ Add launch or other configuration files to Job.

        Files may be specified using relative, absolute, or package-relative
        paths. Files may also be specified using shell globs.

        :param package: Optionally specify a package to search for the file
            or glob relative-to.
        :type package: str
        :param filename: Name of a file to add to the job. Relative to the
            package path, if specified.
        :type filename: str
        :param glob: Shell glob of files to add to the job. Relative to the
            package path, if specified.
        :type glob: str
        """

        if package:
            search_paths = reversed(find_in_workspaces(project=package))
        else:
            search_paths = ('.',)

        if glob and filename:
            raise RuntimeError("You must specify only an exact filename or a glob, not both.")

        # See: https://docs.python.org/2/library/os.html#os.getlogin
        if filename:
            for path in search_paths:
                candidate = os.path.join(path, filename)
                if os.path.isfile(candidate):
                    self.files.append(candidate)

        if glob:
            for path in search_paths:
                self.files.extend(glob_files(os.path.join(path, glob))) 

def setUp(self):
        """
        Defining the exitcodes

        Expected Exitcodes:
        On Unix, the return value is the exit status of the
        process encoded in the format specified for wait()
        https://docs.python.org/3.5/library/os.html#os.system
        """

        self.exit_0 = 0 << 8
        self.exit_1 = 1 << 8
        self.exit_2 = 2 << 8 

def is_posix():
    """
    Return ``True`` if running on a POSIX OS.

    :rtype: bool
    """
    # from https://docs.python.org/2/library/os.html#os.name
    # the registered values of os.name are:
    # "posix", "nt", "os2", "ce", "java", "riscos"
    return os.name == "posix" 

def create_direcotry(path, mode=0o777):
    # please read the following section
    # https://docs.python.org/2/library/os.html#mkdir-modebits
    if not os.path.exists(path):
        try:
            previous_mask = os.umask(0)
            os.makedirs(path, mode=mode)
        except OSError as e:
            logger.error(e.message)
        finally:
            # set the previous mask back
            os.umask(previous_mask) 

def syscall_write( s, arg0, arg1, arg2 ):

  fd       = arg0
  data_ptr = arg1
  nbytes   = arg2

  if s.debug.enabled( "syscalls" ):
    print "syscall_write( fd=%x, buf=%x, count=%x )" % \
          ( fd, data_ptr, nbytes ),

  if not is_fd_open( s, fd ):
    return -1, BAD_FD_ERRNO

  data = get_str( s, data_ptr, nbytes )

  try:
    nbytes_written = os.write( fd, data )
    errno = 0

  except OSError as e:
    if s.debug.enabled( "syscalls" ):
      print "OSError in syscall_write. errno=%d" % e.errno
    nbytes_written = -1
    errno = e.errno

  # https://docs.python.org/2/library/os.html#os.fsync
  #os.fsync( fd )  # this causes Invalid argument error for some reason...

  return nbytes_written, errno

#-----------------------------------------------------------------------
# open
#----------------------------------------------------------------------- 

def copy_stat_to_mem( self, py_stat, mem, addr ):

    # we set the buffer fields one by one. Only copying the fields that
    # are os-independent according to
    # https://docs.python.org/2/library/os.html#os.stat

    self.set_field( Stat.ST_MODE,       py_stat.st_mode    )
    self.set_field( Stat.ST_INO,        py_stat.st_ino     )
    self.set_field( Stat.ST_DEV,        py_stat.st_dev     )
    self.set_field( Stat.ST_NLINK,      py_stat.st_nlink   )
    self.set_field( Stat.ST_UID,        py_stat.st_uid     )
    self.set_field( Stat.ST_GID,        py_stat.st_gid     )
    self.set_field( Stat.ST_SIZE,       py_stat.st_size    )
    # atime, mtime, ctime are floats, so we cast them to int
    self.set_field( Stat.ST_ATIME, int( py_stat.st_atime ) )
    self.set_field( Stat.ST_MTIME, int( py_stat.st_mtime ) )
    self.set_field( Stat.ST_CTIME, int( py_stat.st_ctime ) )

    # now we can copy the buffer

    # TODO: this could be more efficient
    assert addr >= 0
    for i in xrange( Stat.SIZE ):
      mem.write( addr + i, 1, ord( self.buffer[i] ) )

#-------------------------------------------------------------------------
# get_str
#-------------------------------------------------------------------------
# gets the python string from a pointer to the simulated memory. If nchars
# is not provided, reads until a null character. 

def test_filehandles_closed(tmpdir):
    # Check that we don't leak open filehandles
    #
    # This test uses the fact that os.remove fails on windows if the file is in
    # use as a proxy for testing that dlisio doesn't leak filehandles.  From the
    # python docs [1]:
    #
    #   On Windows, attempting to remove a file that is in use causes an
    #   exception to be raised; on Unix, the directory entry is removed but the
    #   storage allocated to the file is not made available until the original
    #   file is no longer in use.
    #
    # On linux on the other hand, os.remove does not fail even if there are
    # open filehandles, hence this test only makes sense on Windows.
    #
    # [1] https://docs.python.org/3/library/os.html

    # Copy the test file to a tmpdir in order to make this test reliable.
    tmp = str(tmpdir.join('206_05a-_3_DWL_DWL_WIRE_258276498.DLIS'))
    shutil.copyfile('data/206_05a-_3_DWL_DWL_WIRE_258276498.DLIS', tmp)

    many_logical = str(tmpdir.join('many_logical'))
    shutil.copyfile('data/chap4-7/many-logical-files.dlis', many_logical)

    with dlisio.load(tmp) as _:
        pass

    with dlisio.load(many_logical) as fls:
        assert len(fls) == 3

    os.remove(tmp)
    os.remove(many_logical) 

def open_atomic(filename, binary=True):
    '''Open a file for atomic writing. Instead of locking this method allows
    you to write the entire file and move it to the actual location. Note that
    this makes the assumption that a rename is atomic on your platform which
    is generally the case but not a guarantee.

    http://docs.python.org/library/os.html#os.rename

    >>> filename = 'test_file.txt'
    >>> if os.path.exists(filename):
    ...     os.remove(filename)

    >>> with open_atomic(filename) as fh:
    ...     written = fh.write(b'test')
    >>> assert os.path.exists(filename)
    >>> os.remove(filename)

    '''
    assert not os.path.exists(filename), '%r exists' % filename
    path, name = os.path.split(filename)

    # Create the parent directory if it doesn't exist
    if path and not os.path.isdir(path):  # pragma: no cover
        os.makedirs(path)

    temp_fh = tempfile.NamedTemporaryFile(
        mode=binary and 'wb' or 'w',
        dir=path,
        delete=False,
    )
    yield temp_fh
    temp_fh.flush()
    os.fsync(temp_fh.fileno())
    temp_fh.close()
    try:
        os.rename(temp_fh.name, filename)
    finally:
        try:
            os.remove(temp_fh.name)
        except Exception:
            pass 

def _verify_fatal(status, output):
  """Check that helper died as expected."""
  # os.abort generates a SIGABRT signal (-6). On Windows, the process
  # immediately returns an exit code of 3.
  # See https://docs.python.org/3.6/library/os.html#os.abort.
  expected_exit_code = 3 if os.name == 'nt' else -6
  _verify_status(expected_exit_code, status, output) 

def test_create_folder(self):
        folder_path = 'new_folder'
        shared_utils.create_folder(folder_path)
        self.assertTrue(os.path.isdir(folder_path))

        # change just-created folder to be unaccessible
        os.chmod(folder_path, 0o0)

        # try creating a folder inside the unaccessible one
        # NOTE: apparently there's no way to make a Windows folder
        # read-only programmatically, so first check if the parent
        # folder cannot be accessed. See Windows note in os.chmod()
        # https://docs.python.org/3/library/os.html#os.chmod
        blocked_folder_path = os.path.join(folder_path, 'blocked_folder')
        if not os.access(folder_path, os.W_OK):
            with self.assertRaises(OSError) as cm:
                shared_utils.create_folder(blocked_folder_path)
            self.assertEqual(cm.exception.errno, 13)

        # try creating a folder with the same name as an existing file
        file_path = 'new_file'
        with open(file_path, "w+"):
            with self.assertRaises(OSError) as cm:
                shared_utils.create_folder(file_path)
            self.assertEqual(cm.exception.errno, 17)

        # remove artifacts
        os.chmod(folder_path, 0o755)
        if os.path.exists(folder_path):
            os.rmdir(folder_path)
        if os.path.exists(file_path):
            os.remove(file_path) 

def _exit_processes(exit_code):  # type: (int) -> None
    """Exit main thread and child processes.

    For more information:
        https://docs.python.org/2/library/os.html#process-management
        https://docs.python.org/3/library/os.html#process-management

    Args:
        exit_code (int): exit code
    """
    os._exit(exit_code)  # pylint: disable=protected-access 

def allocate_shm_file(filename, nbytes, dbytes, readonly):
  exists = os.path.exists(filename)
  size = 0 if not exists else os.path.getsize(filename)

  if readonly and not exists:
    raise SharedMemoryReadError(filename + " has not been allocated. Requested " + str(nbytes) + " bytes.")
  elif readonly and size != nbytes:
    raise SharedMemoryReadError("{} exists, but the allocation size ({} bytes) does not match the request ({} bytes).".format(
      filename, size, nbytes
    ))

  if exists: 
    if size > nbytes:
      with open(filename, 'wb') as f:
        os.ftruncate(f.fileno(), nbytes)
    elif size < nbytes:
      # too small? just remake it below
      os.unlink(filename) 

  exists = os.path.exists(filename)

  if not exists:
    # Previously we were writing out real files full of zeros, 
    # but a) that takes forever and b) modern OSes support sparse
    # files (i.e. gigabytes of zeros that take up only a few real bytes).
    #
    # The following should take advantage of this functionality and be faster.
    # It should work on Python 2.7 Unix, and Python 3.5+ on Unix and Windows.
    #
    # References:
    #   https://stackoverflow.com/questions/8816059/create-file-of-particular-size-in-python
    #   https://docs.python.org/3/library/os.html#os.ftruncate
    #   https://docs.python.org/2/library/os.html#os.ftruncate
    #
    with open(filename, 'wb') as f:
      os.ftruncate(f.fileno(), nbytes) 

def contain(command):
    # TODO: exec command, note the difference between the exec flavours
    #       https://docs.python.org/2/library/os.html#os.execv
    # NOTE: command is an array (the first element is path/file, and the entire
    #       array is exec's args)

    os._exit(0)  # TODO: remove this after adding exec 

def send_sigint(self):
        # `signal.CTRL_C_EVENT` is also sent to the test process itself.
        # See https://docs.python.org/3.6/library/os.html#os.kill
        # So we need to wait the signal and ignore it.
        # We can NOT ignore the signal by modifying the signal handler here.
        # If we temporary ignores the signal, the signal will sent again
        # when the signal handler is restored.
        # If we ignore the signal permanently, we couldn't interrupt the test.
        if os.name == 'nt':
            try:
                os.kill(self.p.pid, signal.CTRL_C_EVENT)
                while True:
                    pass
            except KeyboardInterrupt:
                pass
        else:
            os.kill(self.p.pid, signal.SIGINT) 

def generate_256bit_key():
    """Generate a pseudo-random secure ready-for-crypto-use key.

    Generate it straight using urandom. Proving randomness is impossible, and a good source
    is a hotly debated subject. As always, opinions are welcome but please inform
    yourself first and be prepared to cite a source.

    Further Reading:
    https://docs.python.org/3.5/library/os.html#os.urandom
    https://docs.python.org/2.7/library/os.html#os.urandom
    https://sockpuppet.org/blog/2014/02/25/safely-generate-random-numbers/
    http://www.2uo.de/myths-about-urandom/
    https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Random/__init__.py
    """
    return binascii.hexlify(os.urandom(32)) 

def finalize(self, source=None):
    """Rename `work_dir` to `target_dir` using `os.rename()`.

    :param str source: An optional source offset into the `work_dir`` to use for the atomic
                       update of `target_dir`. By default the whole `work_dir` is used.

    If a race is lost and `target_dir` already exists, the `target_dir` dir is left unchanged and
    the `work_dir` directory will simply be removed.
    """
    if self.is_finalized:
      return

    source = os.path.join(self._work_dir, source) if source else self._work_dir
    try:
      # Perform an atomic rename.
      #
      # Per the docs: https://docs.python.org/2.7/library/os.html#os.rename
      #
      #   The operation may fail on some Unix flavors if src and dst are on different filesystems.
      #   If successful, the renaming will be an atomic operation (this is a POSIX requirement).
      #
      # We have satisfied the single filesystem constraint by arranging the `work_dir` to be a
      # sibling of the `target_dir`.
      os.rename(source, self._target_dir)
    except OSError as e:
      if e.errno not in (errno.EEXIST, errno.ENOTEMPTY):
        raise e
    finally:
      self.cleanup() 

def safe_sleep(seconds):
  """Ensure that the thread sleeps at a minimum the requested seconds.

  Until Python 3.5, there was no guarantee that time.sleep() would actually sleep the requested
  time. See https://docs.python.org/3/library/time.html#time.sleep."""
  if sys.version_info[0:2] >= (3, 5):
    time.sleep(seconds)
  else:
    start_time = current_time = time.time()
    while current_time - start_time < seconds:
      remaining_time = seconds - (current_time - start_time)
      time.sleep(remaining_time)
      current_time = time.time() 

def __import__(*args, **kwargs):
    global _is_debug

    if _is_debug:
        print "*** HOOKED *** IMPORT: args = %s, kwargs = %s" % (args, kwargs)

    name = args[0]

    try:
        return _real_import(*args, **kwargs)
    except ImportError:
        # TODO: Add support for more extensions? (e.g. *.pyc)
        if _mem_storage.has_key(args[0] + '.py'):
            name = args[0] + '.py'
        if _mem_storage.has_key(name):
            # It's a Virtual File!
            new_mod = imp.new_module(name)
            exec _mem_storage[name].read() in new_mod.__dict__
            sys.modules[args[0]] = new_mod
            return new_mod
        else:
            # It's a bust!
            raise ImportError('ImportError: No module named %s' % name)


# TODO: Monkey patch https://docs.python.org/2/library/os.html#file-descriptor-operations 

def read(self, *args, **kwargs):
        _size = None

        try:
            _size = kwargs.get('size', args[0])
        except IndexError:
            pass

        return self.getvalue()[:_size]

    # https://docs.python.org/2/library/stdtypes.html#bltin-file-objects Says file.write(str) 

def test_set_config_value_file_permissions(self):
        self.cli_config.set_value('test_section', 'test_option', 'a_value')
        file_mode = os.stat(self.cli_config.config_path).st_mode
        self.assertTrue(bool(file_mode & stat.S_IRUSR))
        self.assertTrue(bool(file_mode & stat.S_IWUSR))
        # only S_IRUSR and S_IWUSR are supported on Windows: https://docs.python.org/3.8/library/os.html#os.chmod
        if os.name != 'nt':
            self.assertFalse(bool(file_mode & stat.S_IXUSR))
            self.assertFalse(bool(file_mode & stat.S_IRGRP))
            self.assertFalse(bool(file_mode & stat.S_IWGRP))
            self.assertFalse(bool(file_mode & stat.S_IXGRP))
            self.assertFalse(bool(file_mode & stat.S_IROTH))
            self.assertFalse(bool(file_mode & stat.S_IWOTH))
            self.assertFalse(bool(file_mode & stat.S_IXOTH)) 

def xlogdb(self, mode='r'):
        """
        Context manager to access the xlogdb file.

        This method uses locking to make sure only one process is accessing
        the database at a time. The database file will be created
        if it not exists.

        Usage example:

            with server.xlogdb('w') as file:
                file.write(new_line)

        :param str mode: open the file with the required mode
            (default read-only)
        """
        if not os.path.exists(self.config.wals_directory):
            os.makedirs(self.config.wals_directory)
        xlogdb = self.xlogdb_file_name

        with ServerXLOGDBLock(self.config.barman_lock_directory,
                              self.config.name):
            # If the file doesn't exist and it is required to read it,
            # we open it in a+ mode, to be sure it will be created
            if not os.path.exists(xlogdb) and mode.startswith('r'):
                if '+' not in mode:
                    mode = "a%s+" % mode[1:]
                else:
                    mode = "a%s" % mode[1:]

            with open(xlogdb, mode) as f:

                # execute the block nested in the with statement
                try:
                    yield f

                finally:
                    # we are exiting the context
                    # if file is writable (mode contains w, a or +)
                    # make sure the data is written to disk
                    # http://docs.python.org/2/library/os.html#os.fsync
                    if any((c in 'wa+') for c in f.mode):
                        f.flush()
                        os.fsync(f.fileno()) 

def _copyfile(src, dst):
    # shutil.copy seems slow, changing to streaming according to
    # http://stackoverflow.com/questions/22078621/python-how-to-copy-files-fast  # noqa
    # Python 3 hangs using open/write method so we proceed with shutil.copy
    #  and only perform the optimized write for Python 2.
    if (constants.python_version == 3):
        # Do not use copy2(), it will have an issue when copying to a
        #  network/mounted drive.
        # Using copy and manual set_date_from_filename gets the job done.
        # The calling function is responsible for setting the time.
        shutil.copy(src, dst)
        return

    try:
        O_BINARY = os.O_BINARY
    except:
        O_BINARY = 0

    READ_FLAGS = os.O_RDONLY | O_BINARY
    WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
    TEN_MEGABYTES = 10485760
    BUFFER_SIZE = min(TEN_MEGABYTES, os.path.getsize(src))

    try:
        fin = os.open(src, READ_FLAGS)
        stat = os.fstat(fin)
        fout = os.open(dst, WRITE_FLAGS, stat.st_mode)
        for x in iter(lambda: os.read(fin, BUFFER_SIZE), ""):
            os.write(fout, x)
    finally:
        try:
            os.close(fin)
        except:
            pass
        try:
            os.close(fout)
        except:
            pass


# If you want cross-platform overwriting of the destination, 
# use os.replace() instead of rename().
# https://docs.python.org/3/library/os.html#os.rename 

def test_filehandles_closed_when_load_fails(tmpdir):
    # Check that we don't leak open filehandles on failure
    #
    # This test uses the fact that os.remove fails on windows if the file is in
    # use as a proxy for testing that dlisio doesn't leak filehandles. From the
    # python docs [1]:
    #
    #   On Windows, attempting to remove a file that is in use causes an
    #   exception to be raised; on Unix, the directory entry is removed but the
    #   storage allocated to the file is not made available until the original
    #   file is no longer in use.
    #
    # On linux on the other hand, os.remove does not fail even if there are
    # open filehandles, hence this test only makes sense on Windows.
    #
    # [1] https://docs.python.org/3/library/os.html

    # Copy the test files to a tmpdir in order to make this test reliable.
    findvrl = str(tmpdir.join('findvrl'))
    shutil.copyfile('data/chap2/nondlis.txt', findvrl)

    offsets = str(tmpdir.join('offsets'))
    shutil.copyfile('data/chap2/wrong-lrhs.dlis', offsets)

    extract = str(tmpdir.join('extract'))
    shutil.copyfile('data/chap2/padbytes-bad.dlis', extract)

    fdata = str(tmpdir.join('fdata'))
    shutil.copyfile('data/chap3/implicit/fdata-broken-obname.dlis', fdata)

    many_logical = str(tmpdir.join('many_logical'))
    shutil.copyfile('data/chap4-7/many-logical-files-error-in-last.dlis',
                    many_logical)

    # dlisio.load fails at findvrl
    with pytest.raises(RuntimeError):
        _ =  dlisio.load(findvrl)

    # dlisio.load fails at core.findoffsets
    with pytest.raises(RuntimeError):
        _ =  dlisio.load(offsets)

    # dlisio.load fails at core.stream.extract
    with pytest.raises(RuntimeError):
        _ =  dlisio.load(extract)

    # dlisio.load fails at core.findfdata
    with pytest.raises(RuntimeError):
        _ =  dlisio.load(fdata)

    # dlisio.load fails, but 1 LF was already processed successfully
    with pytest.raises(RuntimeError):
        _ =  dlisio.load(many_logical)

    # If dlisio has properly closed the files, removing them should work.
    os.remove(findvrl)
    os.remove(offsets)
    os.remove(extract)
    os.remove(fdata)
    os.remove(many_logical) 

def chmodR(self, perm, target, writemode):
        '''Recursively apply chmod to a directory
        
        @author: Eric Ball

        :param perm: Permissions to be applied. For information on available
                     permissions/modes, see os.chmod documentation at
                     https://docs.python.org/2/library/os.html#os.chmod
        :param target: Target directory
        :param writemode: a]ppend or [o]verwrite

        '''
        try:
            if not os.path.isdir(target):
                raise TypeError(target)
            else:
                try:
                    if writemode[0] == "a":
                        for root, dirs, files in os.walk(target):
                            # Change permissions for root directory
                            currentPerm = os.stat(root)[0]
                            newPerm = currentPerm | perm
                            os.chmod(root, newPerm)
                            # Change permissions for child directories
                            for mydir in dirs:
                                currentPerm = os.stat(os.path.join(root, mydir))[0]
                                newPerm = currentPerm | perm
                                os.chmod(os.path.join(root, mydir), newPerm)
                            # Change permissions for all files
                            for myfile in files:
                                currentPerm = os.stat(os.path.join(root,
                                                                   myfile))[0]
                                newPerm = currentPerm | perm
                                os.chmod(os.path.join(root, myfile), newPerm)
                    elif writemode[0] == "o":
                        for root, dirs, files in os.walk(target):
                            # Change permissions for root directory
                            os.chmod(root, perm)
                            # Change permissions for child directories
                            for mydir in dirs:
                                os.chmod(os.path.join(root, mydir), perm)
                            # Change permissions for all files
                            for myfile in files:
                                os.chmod(os.path.join(root, myfile), perm)
                    else:
                        raise NameError(writemode)
                except NameError:
                    raise
        except TypeError:
            print("Error: Cannot chmodR target, must be a directory")
            raise
        except NameError:
            print(("Error: Invalid writemode specified. Please use [a]ppend " + \
                "or [o]verwrite"))
            raise
        except Exception:
            raise