Python os.fsync() Examples

The following are 30 code examples of os.fsync(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module os , or try the search function .
Example #1
Source File: pytables.py    From Computable with MIT License 6 votes vote down vote up
def flush(self, fsync=False):
        """
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        """
        if self._handle is not None:
            self._handle.flush()
            if fsync:
                try:
                    os.fsync(self._handle.fileno())
                except:
                    pass 
Example #2
Source File: dcos_marathon.py    From ansible-dcos with Apache License 2.0 6 votes vote down vote up
def app_create(app_id, options):
    """Deploy an app via Marathon"""
    display.vvv("DC/OS: Marathon create app {}".format(app_id))

    # create a temporary file for the options json file
    with tempfile.NamedTemporaryFile('w+') as f:
        json.dump(options, f)

        # force write the file to disk to make sure subcommand can read it
        f.flush()
        os.fsync(f)

        display.vvv(subprocess.check_output(
        ['cat', f.name]).decode())

        cmd = [
            'dcos',
            'marathon',
            'app',
            'add',
            f.name
        ]
        run_command(cmd, 'add app', stop_on_error=True) 
Example #3
Source File: utils.py    From nn-compression with MIT License 6 votes vote down vote up
def write(self, content, wrap=True, flush=False, verbose=False):
        """
        write file and flush buffer to the disk
        :param content: str
        :param wrap: bool, whether to add '\n' at the end of the content
        :param flush: bool, whether to flush buffer to the disk, default=False
        :param verbose: bool, whether to print the content, default=False
        :return:
            void
        """
        if verbose:
            print(content)
        if wrap:
            content += "\n"
        self.f.write(content)
        if flush:
            self.f.flush()
            os.fsync(self.fid) 
Example #4
Source File: storage.py    From securesystemslib with MIT License 6 votes vote down vote up
def put(self, fileobj, filepath):
    # If we are passed an open file, seek to the beginning such that we are
    # copying the entire contents
    if not fileobj.closed:
      fileobj.seek(0)

    try:
      with open(filepath, 'wb') as destination_file:
        shutil.copyfileobj(fileobj, destination_file)
        # Force the destination file to be written to disk from Python's internal
        # and the operating system's buffers.  os.fsync() should follow flush().
        destination_file.flush()
        os.fsync(destination_file.fileno())
    except (OSError, IOError):
      raise securesystemslib.exceptions.StorageError(
          "Can't write file %s" % filepath) 
Example #5
Source File: filesystem.py    From pipenv with MIT License 6 votes vote down vote up
def adjacent_tmp_file(path):
    # type: (str) -> Iterator[NamedTemporaryFileResult]
    """Given a path to a file, open a temp file next to it securely and ensure
    it is written to disk after the context reaches its end.
    """
    with NamedTemporaryFile(
        delete=False,
        dir=os.path.dirname(path),
        prefix=os.path.basename(path),
        suffix='.tmp',
    ) as f:
        result = cast('NamedTemporaryFileResult', f)
        try:
            yield result
        finally:
            result.file.flush()
            os.fsync(result.file.fileno()) 
Example #6
Source File: fsutil.py    From pykit with MIT License 6 votes vote down vote up
def write_file(path, fcont, uid=None, gid=None, atomic=False, fsync=True):

    if not atomic:
        return _write_file(path, fcont, uid, gid, fsync)

    tmp_path = '{path}._tmp_.{pid}_{timestamp}'.format(
        path=path,
        pid=os.getpid(),
        timestamp=timeutil.ns(),
    )
    _write_file(tmp_path, fcont, uid, gid, fsync)

    try:
        os.rename(tmp_path, path)
    except EnvironmentError:
        os.remove(tmp_path)
        raise 
Example #7
Source File: pytribe.py    From PyTribe with GNU General Public License v3.0 6 votes vote down vote up
def _log_header(self):

		"""Logs a header to the data file
		"""

		# write a header to the data file
		header = self._separator.join(['timestamp','time','fix','state',
								'rawx','rawy','avgx','avgy','psize',
								'Lrawx','Lrawy','Lavgx','Lavgy','Lpsize','Lpupilx','Lpupily',
								'Rrawx','Rrawy','Ravgx','Ravgy','Rpsize','Rpupilx','Rpupily'
								])
		self._logfile.write(header + '\n') # to internal buffer
		self._logfile.flush() # internal buffer to RAM
		os.fsync(self._logfile.fileno()) # RAM file cache to disk
		self._firstlog = False



# # # # # #
# PARALLEL ClASS


# Ugly, but sod it: A global variable for the most recent sample. 
Example #8
Source File: pytribe.py    From PyTribe with GNU General Public License v3.0 6 votes vote down vote up
def stop_recording(self):

		"""Stops data recording
		"""

		# consolidate the data file on the hard drive
		# internal buffer to RAM
		self._logfile.flush()
		# RAM file cache to disk
		os.fsync(self._logfile.fileno())

		# set self._logdata to False, so the data processing thread does not
		# write samples to the log file
		if self._logdata:
			self.log_message("stop_recording")
			self._logdata = False 
Example #9
Source File: pytables.py    From recruit with Apache License 2.0 6 votes vote down vote up
def flush(self, fsync=False):
        """
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        """
        if self._handle is not None:
            self._handle.flush()
            if fsync:
                try:
                    os.fsync(self._handle.fileno())
                except OSError:
                    pass 
Example #10
Source File: dcos_edgelb.py    From ansible-dcos with Apache License 2.0 6 votes vote down vote up
def pool_update(pool_id, instance_name, options):
    """Update an pool"""
    display.vvv("DC/OS: Edgelb update pool {}".format(pool_id))

    # create a temporary file for the options json file
    with tempfile.NamedTemporaryFile('w+') as f:
        json.dump(options, f)

        # force write the file to disk to make sure subcommand can read it
        f.flush()
        os.fsync(f)

        display.vvv(subprocess.check_output(
        ['cat', f.name]).decode())

        cmd = [
            'dcos',
            'edgelb',
            'update',
            '--name=' + instance_name,
            f.name
        ]
        run_command(cmd, 'update pool', stop_on_error=True) 
Example #11
Source File: dcos_edgelb.py    From ansible-dcos with Apache License 2.0 6 votes vote down vote up
def pool_create(pool_id, instance_name, options):
    """Create a pool"""
    display.vvv("DC/OS: edgelb create pool {}".format(pool_id))

    # create a temporary file for the options json file
    with tempfile.NamedTemporaryFile('w+') as f:
        json.dump(options, f)

        # force write the file to disk to make sure subcommand can read it
        f.flush()
        os.fsync(f)

        display.vvv(subprocess.check_output(
        ['cat', f.name]).decode())

        cmd = [
            'dcos',
            'edgelb',
            'create',
            '--name=' + instance_name,
            f.name
        ]
        run_command(cmd, 'update pool', stop_on_error=True) 
Example #12
Source File: test_gzip.py    From ironpython2 with Apache License 2.0 6 votes vote down vote up
def test_write(self):
        with gzip.GzipFile(self.filename, 'wb') as f:
            f.write(data1 * 50)

            # Try flush and fileno.
            f.flush()
            f.fileno()
            if hasattr(os, 'fsync'):
                os.fsync(f.fileno())
            f.close()

        # Test multiple close() calls.
        f.close()

    # The following test_write_xy methods test that write accepts
    # the corresponding bytes-like object type as input
    # and that the data written equals bytes(xy) in all cases. 
Example #13
Source File: reflink.py    From qubes-core-admin with GNU Lesser General Public License v2.1 6 votes vote down vote up
def _replace_file(dst):
    ''' Yield a tempfile whose name starts with dst, creating the last
        directory component if necessary. If the block does not raise
        an exception, safely rename the tempfile to dst.
    '''
    tmp_dir, prefix = os.path.split(dst + '~')
    _make_dir(tmp_dir)
    tmp = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix=prefix, delete=False)
    try:
        yield tmp
        tmp.flush()
        os.fsync(tmp.fileno())
        tmp.close()
        _rename_file(tmp.name, dst)
    except:
        tmp.close()
        _remove_file(tmp.name)
        raise 
Example #14
Source File: pytables.py    From vnpy_crypto with MIT License 6 votes vote down vote up
def flush(self, fsync=False):
        """
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        """
        if self._handle is not None:
            self._handle.flush()
            if fsync:
                try:
                    os.fsync(self._handle.fileno())
                except:
                    pass 
Example #15
Source File: pd_storage.py    From Paradrop with Apache License 2.0 6 votes vote down vote up
def saveToDisk(self):
        """Saves the data to disk."""
        out.info('Saving to disk (%s)\n' % (self.filename))

        # Make sure they want to save
        if(not self.attrSaveable()):
            return

        # Get whatever the data is
        pyld = self.exportAttr(self.getAttr())

        # Write the file to disk, truncate if it exists
        try:
            with open(self.filename, 'wb') as output:
                pickle.dump(pyld, output)
                os.fsync(output.fileno())
        except Exception as e:
            out.err('Error writing to disk %s\n' % (str(e)))

        try:
            with open(self.filename + ".yaml", "w") as output:
                yaml.dump(pyld, output)
        except Exception as error:
            out.err("Error writing yaml file: {}".format(error)) 
Example #16
Source File: atomic_write.py    From DRL_DeliveryDuel with MIT License 6 votes vote down vote up
def atomic_write(filepath, binary=False, fsync=False):
    """ Writeable file object that atomically updates a file (using a temporary file). In some cases (namely Python < 3.3 on Windows), this could result in an existing file being temporarily unlinked.

    :param filepath: the file path to be opened
    :param binary: whether to open the file in a binary mode instead of textual
    :param fsync: whether to force write the file to disk
    """

    tmppath = filepath + '~'
    while os.path.isfile(tmppath):
        tmppath += '~'
    try:
        with open(tmppath, 'wb' if binary else 'w') as file:
            yield file
            if fsync:
                file.flush()
                os.fsync(file.fileno())
        replace(tmppath, filepath)
    finally:
        try:
            os.remove(tmppath)
        except (IOError, OSError):
            pass 
Example #17
Source File: local_fsclient.py    From a2ml with Apache License 2.0 6 votes vote down vote up
def open_atomic(self, path, mode):
        parent = self.get_parent_folder(os.path.abspath(path))
        self.list_folder(parent)

        temp_dir = self.get_temp_folder()
        try:
            temp_path = os.path.join(temp_dir, os.path.basename(path))
            with open(temp_path, mode) as f:
                try:
                    yield f
                finally:
                    f.flush()  # flush file buffers
                    os.fsync(f.fileno())  # ensure all data are written to disk
            if platform.system() == "Windows":
                if os.path.exists(path):
                    os.remove(path)

            os.rename(temp_path, path)  # atomic move to target place
        finally:
            self.remove_folder(temp_dir) 
Example #18
Source File: local_fsclient.py    From a2ml with Apache License 2.0 6 votes vote down vote up
def write_text_file(self, path, data, atomic=False, mode="w"):
        self.create_parent_folder(path)

        if atomic:
            with self.open_atomic(path, mode) as file:
                file.write(data)
        else:
            from a2ml.api.utils import fsclient
            self.remove_file(path)

            with fsclient.open_file(path, mode) as file:
                try:
                    file.write(data)
                finally:
                    file.flush()  # flush file buffers
                    os.fsync(file.fileno())

        self.read_text_file(path) 
Example #19
Source File: wlt_2_updateconfig.py    From WLANThermo_v2 with GNU General Public License v3.0 6 votes vote down vote up
def config_write(configfile, config, oldconfig):
    # Schreibt das Configfile
    # Ein Lock sollte im aufrufenden Programm gehalten werden!

    tmp_filename = get_random_filename(configfile)
    with codecs.open(tmp_filename, 'w', 'utf_8') as new_ini:
        for section_name in config.sections():
            new_ini.write(u'[{section_name}]\n'.format(section_name=section_name))
            for (key, value) in config.items(section_name):
                try:
                    new_ini.write(u'{key} = {value}\n'.format(key=key, value=update_settings(section_name, key, oldconfig.get(section_name, key))))
                except (configparser.NoSectionError, configparser.NoOptionError):
                    new_ini.write(u'{key} = {value}\n'.format(key=key, value=value))
            new_ini.write('\n')
        new_ini.flush()
        os.fsync(new_ini.fileno())
        new_ini.close()
        os.rename(tmp_filename, configfile) 
Example #20
Source File: maverick.py    From WLANThermo_v2 with GNU General Public License v3.0 6 votes vote down vote up
def json_writer():
   # schreibt ein JSON-Logfile
   if options.verbose:
       print('Starting JSON writer Task')
   if options.fahrenheit:
      unit = '°F'
   else:
      unit = '°C'
   while True:
      item_time, chksum_is, type, temp1, temp2 =  json_queue.get()
      set = {'time': item_time, 'checksum': chksum_is, 'type' : type, 'unit': unit, 'temperature_1' : temp1, 'temperature_2' : temp2}
      if options.noappend:
        tmp_filename = get_random_filename(options.json)
        with open(tmp_filename, 'w') as json_file:
            json_file.write(json.dumps(set))
            json_file.flush()
            os.fsync(json_file.fileno())
            json_file.close()
            os.rename(tmp_filename, options.json)
      else:
        with open(options.json, 'a') as json_file:
            json_file.write(json.dumps(set) + ',')
            json_file.flush()

      json_queue.task_done() 
Example #21
Source File: wallet.py    From torba with MIT License 6 votes vote down vote up
def write(self, json_dict):

        json_data = json.dumps(json_dict, indent=4, sort_keys=True)
        if self.path is None:
            return json_data

        temp_path = "%s.tmp.%s" % (self.path, os.getpid())
        with open(temp_path, "w") as f:
            f.write(json_data)
            f.flush()
            os.fsync(f.fileno())

        if os.path.exists(self.path):
            mode = os.stat(self.path).st_mode
        else:
            mode = stat.S_IREAD | stat.S_IWRITE
        try:
            os.rename(temp_path, self.path)
        except Exception:  # pylint: disable=broad-except
            os.remove(self.path)
            os.rename(temp_path, self.path)
        os.chmod(self.path, mode) 
Example #22
Source File: boot_config.py    From kano-settings with GNU General Public License v2.0 6 votes vote down vote up
def _remove_noobs_defaults(self):
        """
        Remove the config entries added by Noobs,
        by removing all the lines after and including
        noobs' sentinel

        """
        lines = read_file_contents_as_lines(self.path)
        with open_locked(self.path, 'w') as boot_config_file:

            for line in lines:
                if line == noobs_line:
                    break

                boot_config_file.write(line + "\n")

            # flush changes to disk
            boot_config_file.flush()
            os.fsync(boot_config_file.fileno()) 
Example #23
Source File: boot_config.py    From kano-settings with GNU General Public License v2.0 6 votes vote down vote up
def set_value(self, name, value=None, config_filter=Filter.ALL):
        # if the value argument is None, the option will be commented out
        lines = read_file_contents_as_lines(self.path)
        if not lines:  # this is true if the file is empty, not sure that was intended.
            return

        logger.info('writing value to {} {} {}'.format(self.path, name, value))

        config = BootConfigParser(lines)
        config.set(name, value, config_filter=config_filter)

        with open_locked(self.path, "w") as boot_config_file:
            boot_config_file.write(config.dump())

            # flush changes to disk
            boot_config_file.flush()
            os.fsync(boot_config_file.fileno()) 
Example #24
Source File: boot_config.py    From kano-settings with GNU General Public License v2.0 6 votes vote down vote up
def close(self):
        if self.state == 2:
            if dry_run:
                logger.info("dry run config transaction can be found in {}".format(self.temp_path))
            else:
                logger.info("closing config transaction")
                shutil.move(self.temp_path, self.path)
                # sync
                dirfd = os.open(self.dir, os.O_DIRECTORY)
                os.fsync(dirfd)
                os.close(dirfd)
                os.system('sync')

        else:
            logger.warn("closing config transaction with no edits")
        self.set_state_idle() 
Example #25
Source File: filesystem.py    From pex with Apache License 2.0 6 votes vote down vote up
def adjacent_tmp_file(path):
    # type: (str) -> Iterator[NamedTemporaryFileResult]
    """Given a path to a file, open a temp file next to it securely and ensure
    it is written to disk after the context reaches its end.
    """
    with NamedTemporaryFile(
        delete=False,
        dir=os.path.dirname(path),
        prefix=os.path.basename(path),
        suffix='.tmp',
    ) as f:
        result = cast('NamedTemporaryFileResult', f)
        try:
            yield result
        finally:
            result.file.flush()
            os.fsync(result.file.fileno()) 
Example #26
Source File: systemctl3.py    From vanilla-docker with MIT License 6 votes vote down vote up
def read_log_files(self, units):
        BUFSIZE=8192
        for unit in units:
            if unit in self._log_file:
                new_text = b""
                while True:
                    buf = os.read(self._log_file[unit], BUFSIZE)
                    if not buf: break
                    new_text += buf
                    continue
                text = self._log_hold[unit] + new_text
                if not text: continue
                lines = text.split(b"\n")
                if not text.endswith(b"\n"):
                    self._log_hold[unit] = lines[-1]
                    lines = lines[:-1]
                for line in lines:
                    prefix = unit.encode("utf-8")
                    content = prefix+b": "+line+b"\n"
                    os.write(1, content)
                    try: os.fsync(1)
                    except: pass 
Example #27
Source File: test_gzip.py    From oss-ftp with MIT License 5 votes vote down vote up
def test_write(self):
        with gzip.GzipFile(self.filename, 'wb') as f:
            f.write(data1 * 50)

            # Try flush and fileno.
            f.flush()
            f.fileno()
            if hasattr(os, 'fsync'):
                os.fsync(f.fileno())
            f.close()

        # Test multiple close() calls.
        f.close() 
Example #28
Source File: __init__.py    From mlimages with MIT License 5 votes vote down vote up
def download_dataset(self, url, relative):
        r = requests.get(url, stream=True)

        if r.ok:
            with self.file_api.open_with_mkdir(relative) as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk:
                        f.write(chunk)
                        f.flush()
                        os.fsync(f.fileno()) 
Example #29
Source File: fsutil.py    From pykit with MIT License 5 votes vote down vote up
def _write_file(path, fcont, uid=None, gid=None, fsync=True):

    uid = uid or config.uid
    gid = gid or config.gid

    with open(path, 'w') as f:
        f.write(fcont)
        f.flush()
        if fsync:
            os.fsync(f.fileno())

    if uid is not None and gid is not None:
        os.chown(path, uid, gid) 
Example #30
Source File: utils.py    From alibabacloud-quantization-networks with Apache License 2.0 5 votes vote down vote up
def flush(self):
        self.console.flush()
        if self.file is not None:
            self.file.flush()
            os.fsync(self.file.fileno())