Python os.fsync() Examples

The following are code examples for showing how to use os.fsync(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don't like. You can also save this page to your account.

Example 1
Project: rca-evaluation   Author: sieve-microservices   File: metadata.py    (license) View Source Project 6 votes vote down vote up
def _atomic_write(filename):
    path = os.path.dirname(filename)
    try:
        file = tempfile.NamedTemporaryFile(delete=False, dir=path, mode="w+")
        yield file
        file.flush()
        os.fsync(file.fileno())
        os.rename(file.name, filename)
    finally:
        try:
            os.remove(file.name)
        except OSError as e:
            if e.errno == 2:
                pass
            else:
                raise e 
Example 2
Project: bob   Author: BobBuildTool   File: state.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def __save(self):
        if self.__asynchronous == 0:
            state = {
                "version" : _BobState.CUR_VERSION,
                "byNameDirs" : self.__byNameDirs,
                "results" : self.__results,
                "inputs" : self.__inputs,
                "jenkins" : self.__jenkins,
                "dirStates" : self.__dirStates,
                "buildState" : self.__buildState,
            }
            tmpFile = self.__path+".new"
            try:
                with open(tmpFile, "wb") as f:
                    pickle.dump(state, f)
                    f.flush()
                    os.fsync(f.fileno())
                os.replace(tmpFile, self.__path)
            except OSError as e:
                raise ParseError("Error saving workspace state: " + str(e))
            self.__dirty = False
        else:
            self.__dirty = True 
Example 3
Project: farfetchd   Author: isislovecruft   File: crypto.py    (license) View Source Project 6 votes vote down vote up
def writeKeyToFile(key, filename):
    """Write **key** to **filename**, with ``0400`` permissions.

    If **filename** doesn't exist, it will be created. If it does exist
    already, and is writable by the owner of the current process, then it will
    be truncated to zero-length and overwritten.

    :param bytes key: A key (or some other private data) to write to
        **filename**.
    :param str filename: The path of the file to write to.
    :raises: Any exceptions which may occur.
    """
    logging.info("Writing key to file: %r", filename)
    flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0)
    fd = os.open(filename, flags, 0400)
    os.write(fd, key)
    os.fsync(fd)
    os.close(fd) 
Example 4
Project: zeronet-debian   Author: bashrc   File: helper.py    (license) View Source Project 6 votes vote down vote up
def atomicWrite(dest, content, mode="w"):
    try:
        with open(dest + "-new", mode) as f:
            f.write(content)
            f.flush()
            os.fsync(f.fileno())
        if os.path.isfile(dest + "-old"):  # Previous incomplete write
            os.rename(dest + "-old", dest + "-old-%s" % time.time())
        os.rename(dest, dest + "-old")
        os.rename(dest + "-new", dest)
        os.unlink(dest + "-old")
        return True
    except Exception, err:
        from Debug import Debug
        logging.error(
            "File %s write failed: %s, reverting..." %
            (dest, Debug.formatException(err))
        )
        if os.path.isfile(dest + "-old") and not os.path.isfile(dest):
            os.rename(dest + "-old", dest)
        return False 
Example 5
Project: PiBunny   Author: tholum   File: smbserver.py    (license) View Source Project 6 votes vote down vote up
def smb2Flush(connId, smbServer, recvPacket):
        connData = smbServer.getConnectionData(connId)

        respSMBCommand = smb2.SMB2Flush_Response()
        flushRequest   = smb2.SMB2Flush(recvPacket['Data'])

        if connData['OpenedFiles'].has_key(str(flushRequest['FileID'])):
             fileHandle = connData['OpenedFiles'][str(flushRequest['FileID'])]['FileHandle']
             errorCode = STATUS_SUCCESS
             try:
                 os.fsync(fileHandle)
             except Exception, e:
                 smbServer.log("SMB2_FLUSH %s" % e, logging.ERROR)
                 errorCode = STATUS_ACCESS_DENIED
        else:
            errorCode = STATUS_INVALID_HANDLE

        smbServer.setConnectionData(connId, connData)
        return [respSMBCommand], None, errorCode 
Example 6
Project: pipper   Author: sernst   File: downloader.py    (license) View Source Project 6 votes vote down vote up
def save(url: str, local_path: str) -> str:
    """ 
    """

    with closing(requests.get(url, stream=True)) as response:
        if response.status_code != 200:
            print((
                '[ERROR]: Unable to download remote package. Has your'
                'authorized URL expired? Is there internet connectivity?'
            ))

        with open(local_path, 'wb') as f:
            for chunk in response:
                f.write(chunk)
                f.flush()
                os.fsync(f.fileno())

    return local_path 
Example 7
Project: geepee   Author: thangbui   File: run_sparse_cla.py    (license) View Source Project 6 votes vote down vote up
def callback(params, iteration, args):
        global X_test
        global y_test
        if (iteration + 1) in test_iters:
            idx = np.where(test_iters == (iteration + 1))[0][0]
            outfile1 = outfile1s[idx]
            outfile2 = outfile2s[idx]
            params_dict = unflatten_dict(params, args[0])
            model.update_hypers(params_dict)
            # We make predictions for the test set
            mf, vf = model.predict_f(X_test)
            mf, vf = mf[:, 0], vf[:, 0]
            # We compute the test error and log lik
            test_nll = compute_nll(y_test, mf, vf, 'cdf')
            outfile2.write('%.6f\n' % test_nll)
            outfile2.flush()
            os.fsync(outfile2.fileno())

            test_error = compute_error(y_test, mf, vf, 'cdf')
            outfile1.write('%.6f\n' % test_error)
            outfile1.flush()
            os.fsync(outfile1.fileno())
    
    # train 
Example 8
Project: geepee   Author: thangbui   File: run_sparse_cla.py    (license) View Source Project 6 votes vote down vote up
def callback(params, iteration, args):
        global X_test
        global y_test
        if (iteration + 1) in test_iters:
            idx = np.where(test_iters == (iteration + 1))[0][0]
            outfile1 = outfile1s[idx]
            outfile2 = outfile2s[idx]
            params_dict = unflatten_dict(params, args[0])
            model.update_hypers(params_dict)
            # We make predictions for the test set
            mf, vf = model.predict_f(X_test)
            mf, vf = mf[:, 0], vf[:, 0]
            # We compute the test error and log lik
            test_nll = compute_nll(y_test, mf, vf, 'cdf')
            outfile2.write('%.6f\n' % test_nll)
            outfile2.flush()
            os.fsync(outfile2.fileno())

            test_error = compute_error(y_test, mf, vf, 'cdf')
            outfile1.write('%.6f\n' % test_error)
            outfile1.flush()
            os.fsync(outfile1.fileno())
    
    # train 
Example 9
Project: tmux2html   Author: tweekmonster   File: main.py    (license) View Source Project 6 votes vote down vote up
def atomic_output(output, filename=None, mode=0o0644, quiet=False):
    if filename:
        tmp = None
        try:
            tmp = tempfile.NamedTemporaryFile(prefix='tmp2html.',
                                              dir=os.path.dirname(filename),
                                              delete=False)
            tmp.write(output.encode('utf8'))
            tmp.flush()
            os.fsync(tmp.fileno())
        except IOError as e:
            print(e)
        except Exception:
            pass
        finally:
            if tmp:
                tmp.close()
                os.chmod(tmp.name, mode)
                os.rename(tmp.name, filename)
                if not quiet:
                    print('Wrote HTML to: {}'.format(filename))
    else:
        print(output.encode('utf8')) 
Example 10
Project: gym   Author: openai   File: atomic_write.py    (license) View Source Project 6 votes vote down vote up
def atomic_write(filepath, binary=False, fsync=False):
    """ Writeable file object that atomically updates a file (using a temporary file). In some cases (namely Python < 3.3 on Windows), this could result in an existing file being temporarily unlinked.

    :param filepath: the file path to be opened
    :param binary: whether to open the file in a binary mode instead of textual
    :param fsync: whether to force write the file to disk
    """

    tmppath = filepath + '~'
    while os.path.isfile(tmppath):
        tmppath += '~'
    try:
        with open(tmppath, 'wb' if binary else 'w') as file:
            yield file
            if fsync:
                file.flush()
                os.fsync(file.fileno())
        replace(tmppath, filepath)
    finally:
        try:
            os.remove(tmppath)
        except (IOError, OSError):
            pass 
Example 11
Project: doorman   Author: mwielgoszewski   File: logstash.py    (license) View Source Project 6 votes vote down vote up
def handle_result(self, data, **kwargs):
        if self.fp is None:
            return

        fp = self.fp

        host_identifier = kwargs.get('host_identifier')
        created = dt.datetime.utcnow().isoformat()

        try:
            for item in extract_results(data):
                json_dump({
                    '@version': 1,
                    '@host_identifier': host_identifier,
                    '@timestamp': item.timestamp.isoformat(),
                    'log_type': 'result',
                    'action': item.action,
                    'columns': item.columns,
                    'name': item.name,
                    'created': created,
                }, fp)
                fp.write('\r\n')
        finally:
            fp.flush()
            os.fsync(fp.fileno()) 
Example 12
Project: oil   Author: oilshell   File: test_gzip.py    (license) View Source Project 6 votes vote down vote up
def test_write(self):
        with gzip.GzipFile(self.filename, 'wb') as f:
            f.write(data1 * 50)

            # Try flush and fileno.
            f.flush()
            f.fileno()
            if hasattr(os, 'fsync'):
                os.fsync(f.fileno())
            f.close()

        # Test multiple close() calls.
        f.close()

    # The following test_write_xy methods test that write accepts
    # the corresponding bytes-like object type as input
    # and that the data written equals bytes(xy) in all cases. 
Example 13
Project: python2-tracer   Author: extremecoders-re   File: test_gzip.py    (license) View Source Project 6 votes vote down vote up
def test_write(self):
        with gzip.GzipFile(self.filename, 'wb') as f:
            f.write(data1 * 50)

            # Try flush and fileno.
            f.flush()
            f.fileno()
            if hasattr(os, 'fsync'):
                os.fsync(f.fileno())
            f.close()

        # Test multiple close() calls.
        f.close()

    # The following test_write_xy methods test that write accepts
    # the corresponding bytes-like object type as input
    # and that the data written equals bytes(xy) in all cases. 
Example 14
Project: studio   Author: studioml   File: auth.py    (license) View Source Project 6 votes vote down vote up
def refresh_token(self, email, refresh_token):
        api_key = os.path.join(TOKEN_DIR, self.firebase.api_key)
        self.user = self.firebase.auth().refresh(refresh_token)
        self.user['email'] = email
        self.user['expiration'] = time.time() + API_KEY_COOLDOWN
        self.expired = False

        # if not os.path.exists(api_key) or \
        #   time.time() - os.path.getmtime(api_key) > HALF_HOUR:
        # Rename to ensure atomic writes to json file
        # (technically more safe, but slower)

        tmp_api_key = os.path.join(tempfile.gettempdir(),
                                   "api_key_%s" % rand_string(32))
        with open(tmp_api_key, 'w') as f:
            f.write(json.dumps(self.user))
            f.flush()
            os.fsync(f.fileno())
            f.close()
        os.rename(tmp_api_key, api_key) 
Example 15
Project: CVE-2017-7494   Author: joxeankoret   File: smbserver.py    (license) View Source Project 6 votes vote down vote up
def smb2Flush(connId, smbServer, recvPacket):
        connData = smbServer.getConnectionData(connId)

        respSMBCommand = smb2.SMB2Flush_Response()
        flushRequest   = smb2.SMB2Flush(recvPacket['Data'])

        if connData['OpenedFiles'].has_key(str(flushRequest['FileID'])):
             fileHandle = connData['OpenedFiles'][str(flushRequest['FileID'])]['FileHandle']
             errorCode = STATUS_SUCCESS
             try:
                 os.fsync(fileHandle)
             except Exception, e:
                 smbServer.log("SMB2_FLUSH %s" % e, logging.ERROR)
                 errorCode = STATUS_ACCESS_DENIED
        else:
            errorCode = STATUS_INVALID_HANDLE

        smbServer.setConnectionData(connId, connData)
        return [respSMBCommand], None, errorCode 
Example 16
Project: pysport   Author: sportorg   File: sireader.py    (license) View Source Project 6 votes vote down vote up
def _send_command(self, command, parameters):
        try:
            if self._serial.inWaiting() != 0:
                raise SIReaderException(
                    'Input buffer must be empty before sending command. Currently %s bytes in the input buffer.' % self._serial.inWaiting())
            command_string = command + int2byte(len(parameters)) + parameters
            crc = SIReader._crc(command_string)
            cmd = SIReader.STX + command_string + crc + SIReader.ETX
            if self._debug:
                print("==>> command '%s', parameters %s, crc %s" % (hexlify(command).decode('ascii'),
                                                                    ' '.join(
                                                                        [hexlify(int2byte(c)).decode('ascii') for c in
                                                                         parameters]),
                                                                    hexlify(crc).decode('ascii'),
                                                                    ))
            self._serial.write(cmd)
        except (SerialException, OSError) as  msg:
            raise SIReaderException('Could not send command: %s' % msg)

        if self._logfile:
            self._logfile.write('s %s %s\n' % (datetime.now(), cmd))
            self._logfile.flush()
            os.fsync(self._logfile)
        return self._read_command() 
Example 17
Project: lbryum   Author: lbryio   File: wallet.py    (license) View Source Project 6 votes vote down vote up
def _write(self):
        if threading.currentThread().isDaemon():
            log.warning('daemon thread cannot write wallet')
            return
        if not self.modified:
            return
        s = json.dumps(self.data, indent=4, sort_keys=True)
        temp_path = "%s.tmp.%s" % (self.path, os.getpid())
        with open(temp_path, "w") as f:
            f.write(s)
            f.flush()
            os.fsync(f.fileno())

        if os.path.exists(self.path):
            mode = os.stat(self.path).st_mode
        else:
            mode = stat.S_IREAD | stat.S_IWRITE
        # perform atomic write on POSIX systems
        try:
            os.rename(temp_path, self.path)
        except:
            os.remove(self.path)
            os.rename(temp_path, self.path)
        os.chmod(self.path, mode)
        self.modified = False 
Example 18
Project: PyDataLondon29-EmbarrassinglyParallelDAWithAWSLambda   Author: SignalMedia   File: pytables.py    (license) View Source Project 6 votes vote down vote up
def flush(self, fsync=False):
        """
        Force all buffered modifications to be written to disk.

        Parameters
        ----------
        fsync : bool (default False)
          call ``os.fsync()`` on the file handle to force writing to disk.

        Notes
        -----
        Without ``fsync=True``, flushing may not guarantee that the OS writes
        to disk. With fsync, the operation will block until the OS claims the
        file has been written; however, other caching layers may still
        interfere.
        """
        if self._handle is not None:
            self._handle.flush()
            if fsync:
                try:
                    os.fsync(self._handle.fileno())
                except:
                    pass 
Example 19
Project: pyjam   Author: 10se1ucgo   File: ffmpeg.py    (license) View Source Project 6 votes vote down vote up
def run(self):
        file_size_dl = 0
        response = requests.get(self.url, stream=True)
        data_chunks = response.iter_content(chunk_size=1024)

        if not os.path.exists('bin'):
            os.mkdir('bin')

        with open('bin/ffmpeg.7z', 'wb') as f:
            while not self.is_aborted():
                try:
                    chunk = next(data_chunks)
                    file_size_dl += len(chunk)
                    logger.info("FFmpeg downloader: Downloaded chunk: {chunk}".format(chunk=len(chunk)))
                    logger.info("FFmpeg downloader: Total downloaded so far: {total}".format(total=file_size_dl))
                    logger.info("FFmpeg downloader: Remaining: {r}".format(r=self.file_size - file_size_dl))
                    if chunk:
                        f.write(chunk)
                        f.flush()
                        # This makes the download super slow.
                        # os.fsync(f.fileno())
                    wx.CallAfter(self.parent.ff_update, message=file_size_dl)
                except StopIteration:
                    wx.CallAfter(self.parent.ff_complete)
                    break 
Example 20
Project: find_circ2   Author: rajewsky-lab   File: find_circ.py    (license) View Source Project 6 votes vote down vote up
def store_index(self,ipath):
        self.logger.info("# indexed_fasta.store_index('%s')" % ipath)
        
        # write to tmp-file first and in the end rename in order to have this atomic 
        # otherwise parallel building of the same index may screw it up.
        
        import tempfile
        tmp = tempfile.NamedTemporaryFile(mode="w",dir = os.path.dirname(ipath),delete=False)
        for chrom in sorted(self.chrom_stats.keys()):
            ofs,ldata,skip,skipchar,size = self.chrom_stats[chrom]
            tmp.write("%s\t%d\t%d\t%d\t%r\t%d\n" % (chrom,ofs,ldata,skip,skipchar,size))
        
        # make sure everything is on disk
        os.fsync(tmp)
        tmp.close()
        
        # make it accessible to everyone
        import stat
        os.chmod(tmp.name, stat.S_IROTH | stat.S_IRGRP | stat.S_IRUSR)
        
        # this is atomic on POSIX as we have created tmp in the same directory, 
        # therefore same filesystem
        os.rename(tmp.name,ipath) 
Example 21
Project: find_circ   Author: rajewsky-lab   File: find_circ.py    (license) View Source Project 6 votes vote down vote up
def store_index(self,ipath):
        debug("# indexed_fasta.store_index('%s')" % ipath)
        
        # write to tmp-file first and in the end rename in order to have this atomic 
        # otherwise parallel building of the same index may screw it up.
        
        import tempfile
        tmp = tempfile.NamedTemporaryFile(mode="w",dir = os.path.dirname(ipath),delete=False)
        for chrom in sorted(self.chrom_stats.keys()):
            ofs,ldata,skip,skipchar,size = self.chrom_stats[chrom]
            tmp.write("%s\t%d\t%d\t%d\t%r\t%d\n" % (chrom,ofs,ldata,skip,skipchar,size))
        
        # make sure everything is on disk
        os.fsync(tmp)
        tmp.close()
        
        # make it accessible to everyone
        import stat
        os.chmod(tmp.name, stat.S_IROTH | stat.S_IRGRP | stat.S_IRUSR)
        
        # this is atomic on POSIX as we have created tmp in the same directory, 
        # therefore same filesystem
        os.rename(tmp.name,ipath) 
Example 22
Project: labgrid   Author: labgrid-project   File: usbstorage.py    (license) View Source Project 6 votes vote down vote up
def write_image(self, filename):
        with open(filename, 'rb') as src, \
                open(self.storage.path, 'wb') as dst:
            src.seek(0, os.SEEK_END)
            size = src.tell()
            src.seek(0, os.SEEK_SET)

            count = 0
            stat = time() + 3
            while True:
                data = src.read(1024*1024)
                if not data:
                    break
                dst.write(data)
                dst.flush()
                os.fsync(dst.fileno())
                count += len(data)
                if time() > stat:
                    stat += 3
                    print("writing image {:.0%}".format(count/size))
            dst.flush()
            os.fsync(dst.fileno()) 
Example 23
Project: bmap-tools   Author: intel   File: BmapCopy.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync=True, verify=True):
        """
        The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time.
        """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 24
Project: bmap-tools   Author: intel   File: BmapCopy2_3.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 25
Project: bmap-tools   Author: intel   File: BmapCopy3_0.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync=True, verify=True):
        """
        The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time.
        """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 26
Project: bmap-tools   Author: intel   File: BmapCopy2_6.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync=True, verify=True):
        """
        The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time.
        """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 27
Project: bmap-tools   Author: intel   File: BmapCopy2_1.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 28
Project: bmap-tools   Author: intel   File: BmapCopy2_0.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 29
Project: bmap-tools   Author: intel   File: BmapCopy2_4.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 30
Project: bmap-tools   Author: intel   File: BmapCopy2_2.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 31
Project: bmap-tools   Author: intel   File: BmapCopy2_5.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync=True, verify=True):
        """
        The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time.
        """

        self._tune_block_device()

        try:
            BmapCopy.copy(self, sync, verify)
        except:
            raise
        finally:
            self._restore_bdev_settings() 
Example 32
Project: bmap-tools   Author: intel   File: BmapCopy1_0.py    (license) View Source Project 6 votes vote down vote up
def copy(self, sync = True, verify = True):
        """ The same as in the base class but tunes the block device for better
        performance before starting writing. Additionally, it forces block
        device synchronization from time to time in order to make sure we do
        not get stuck in 'fsync()' for too long time. The problem is that the
        kernel synchronizes block devices when the file is closed. And the
        result is that if the user interrupts us while we are copying the data,
        the program will be blocked in 'close()' waiting for the block device
        synchronization, which may last minutes for slow USB stick. This is
        very bad user experience, and we work around this effect by
        synchronizing from time to time. """

        try:
            self._tune_block_device()
            BmapCopy.copy(self, sync, verify)
        except:
            self._restore_bdev_settings()
            raise 
Example 33
Project: unitils   Author: iLoveTux   File: util.py    (license) View Source Project 6 votes vote down vote up
def make_test_data_directory(root="."):
    tmp_dir = os.path.join(root, "test-data")
    if os.path.exists(tmp_dir):
        shutil.rmtree(tmp_dir)
    os.mkdir(tmp_dir)
    os.makedirs(os.path.join(tmp_dir, "branch-1", "level-2", "test"))
    filenames = [
        os.path.join(tmp_dir, "test.txt"),
        os.path.join(tmp_dir, "branch-1", "test.txt"),
        os.path.join(tmp_dir, "branch-1", "level-2", "test.txt"),
        os.path.join(tmp_dir, "branch-1", "level-2", "test", "test.txt"),
        os.path.join(tmp_dir, "Test_.txt"),
        os.path.join(tmp_dir, "branch-1", "Test_.txt"),
        os.path.join(tmp_dir, "branch-1", "level-2", "Test_.txt"),
        os.path.join(tmp_dir, "branch-1", "level-2", "test", "Test_.txt"),
    ]
    for filename in filenames:
        with open(filename, "w") as fp:
            fp.write("Hello, world from: {}\n".format(filename))
            fp.flush()
            os.fsync(fp.fileno())
    return tmp_dir 
Example 34
Project: kekescan   Author: xiaoxiaoleo   File: smbserver.py    (license) View Source Project 6 votes vote down vote up
def smb2Flush(connId, smbServer, recvPacket):
        connData = smbServer.getConnectionData(connId)

        respSMBCommand = smb2.SMB2Flush_Response()
        flushRequest   = smb2.SMB2Flush(recvPacket['Data'])

        if connData['OpenedFiles'].has_key(str(flushRequest['FileID'])):
             fileHandle = connData['OpenedFiles'][str(flushRequest['FileID'])]['FileHandle']
             errorCode = STATUS_SUCCESS
             try:
                 os.fsync(fileHandle)
             except Exception, e:
                 smbServer.log("SMB2_FLUSH %s" % e, logging.ERROR)
                 errorCode = STATUS_ACCESS_DENIED
        else:
            errorCode = STATUS_INVALID_HANDLE

        smbServer.setConnectionData(connId, connData)
        return [respSMBCommand], None, errorCode 
Example 35
Project: ouroboros   Author: pybee   File: test_gzip.py    (license) View Source Project 6 votes vote down vote up
def test_write(self):
        with gzip.GzipFile(self.filename, 'wb') as f:
            f.write(data1 * 50)

            # Try flush and fileno.
            f.flush()
            f.fileno()
            if hasattr(os, 'fsync'):
                os.fsync(f.fileno())
            f.close()

        # Test multiple close() calls.
        f.close()

    # The following test_write_xy methods test that write accepts
    # the corresponding bytes-like object type as input
    # and that the data written equals bytes(xy) in all cases. 
Example 36
Project: AI-Fight-the-Landlord   Author: YoungGer   File: atomic_write.py    (license) View Source Project 6 votes vote down vote up
def atomic_write(filepath, binary=False, fsync=False):
    """ Writeable file object that atomically updates a file (using a temporary file). In some cases (namely Python < 3.3 on Windows), this could result in an existing file being temporarily unlinked.

    :param filepath: the file path to be opened
    :param binary: whether to open the file in a binary mode instead of textual
    :param fsync: whether to force write the file to disk
    """

    tmppath = filepath + '~'
    while os.path.isfile(tmppath):
        tmppath += '~'
    try:
        with open(tmppath, 'wb' if binary else 'w') as file:
            yield file
            if fsync:
                file.flush()
                os.fsync(file.fileno())
        replace(tmppath, filepath)
    finally:
        try:
            os.remove(tmppath)
        except (IOError, OSError):
            pass 
Example 37
Project: rdiff-backup   Author: sol1   File: rpath.py    (license) View Source Project 6 votes vote down vote up
def delete(self):
		"""Delete file at self.path.  Recursively deletes directories."""
		log.Log("Deleting %s" % self.path, 7)
		if self.isdir():
			try: self.rmdir()
			except os.error:
				if Globals.fsync_directories: self.fsync()
				self.conn.shutil.rmtree(self.path)
		else:
			try: self.conn.os.unlink(self.path)
			except OSError, error:
				if error.errno in (errno.EPERM, errno.EACCES):
					# On Windows, read-only files cannot be deleted.
					# Remove the read-only attribute and try again.
					self.chmod(0700)
					self.conn.os.unlink(self.path)
				else:
					raise

		self.setdata() 
Example 38
Project: rabbit2ev   Author: lovelle   File: nullmailer.py    (license) View Source Project 6 votes vote down vote up
def fsync_pool(self):
        """
        Call fsync() on the queue directory
        """
        fd = -1
        if not os.path.isdir(self.queue) or not os.access(self.queue, os.R_OK):
            raise NullMailerErrorPool(
                "%s' not exist or inaccesible" % self.queue)

        try:
            fd = os.open(self.queue, os.O_RDONLY)
            os.fsync(fd)
        except Exception as e:
            raise NullMailerErrorPool(e)
        finally:
            if fd > -1:
                os.close(fd) 
Example 39
Project: dcos-tunnel   Author: dcos   File: cli.py    (license) View Source Project 6 votes vote down vote up
def logging_exec(ssh_client, ssh_command, outputlog, raw=False):
    """
    raw: If True, treat outputlog as a file descriptor
    """
    # This doesn't use paramiko.SSHClient.exec_command() because that doesn't
    # expose the option to combine stderr and stdout
    chan = ssh_client.get_transport().open_session()
    chan.get_pty()
    chan.exec_command(ssh_command)
    chan.set_combine_stderr(True)
    stdout = chan.makefile('r', -1)

    # It's not using `for line in stdout.readlines()` because it's broken
    # in paramiko. It blocks until EOF is reached to output anything.
    line = stdout.readline()
    while line != "":
        if raw:
            os.write(outputlog, line.encode(sys.getdefaultencoding()))
            os.fsync(outputlog)
        else:
            outputlog.write(line)
            outputlog.flush()
        line = stdout.readline() 
Example 40
Project: localstack   Author: localstack   File: common.py    (license) View Source Project 6 votes vote down vote up
def download(url, path, verify_ssl=True):
    """Downloads file at url to the given path"""
    # make sure we're creating a new session here to
    # enable parallel file downloads during installation!
    s = requests.Session()
    r = s.get(url, stream=True, verify=verify_ssl)
    total = 0
    try:
        if not os.path.exists(os.path.dirname(path)):
            os.makedirs(os.path.dirname(path))
        LOGGER.debug('Starting download from %s to %s (%s bytes)' % (url, path, r.headers.get('content-length')))
        with open(path, 'wb') as f:
            for chunk in r.iter_content(DOWNLOAD_CHUNK_SIZE):
                total += len(chunk)
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    LOGGER.debug('Writing %s bytes (total %s) to %s' % (len(chunk), total, path))
                else:
                    LOGGER.debug('Empty chunk %s (total %s) from %s' % (chunk, total, url))
            f.flush()
            os.fsync(f)
    finally:
        LOGGER.debug('Done downloading %s, response code %s' % (url, r.status_code))
        r.close()
        s.close() 
Example 41
Project: vim-clangd   Author: Chilledheart   File: binary_downloader.py    (license) View Source Project 6 votes vote down vote up
def HashFile(file_path, algorithm):
    if not algorithm in ['md5', 'sha1']:
        return None
    with open(file_path, 'rb') as f:
        # FIXME is it necessary?
        # osx get wrong result if not put in the same time
        if sys_platform != 'msys' and sys_platform != 'win32':
            os.fsync(f.fileno())
        h = hashlib.new(algorithm)
        while True:
            data = f.read(4096)
            if not data:
                break
            h.update(data)
        return h.hexdigest()
    return None 
Example 42
Project: gym-adv   Author: lerrel   File: atomic_write.py    (license) View Source Project 6 votes vote down vote up
def atomic_write(filepath, binary=False, fsync=False):
    """ Writeable file object that atomically updates a file (using a temporary file). In some cases (namely Python < 3.3 on Windows), this could result in an existing file being temporarily unlinked.

    :param filepath: the file path to be opened
    :param binary: whether to open the file in a binary mode instead of textual
    :param fsync: whether to force write the file to disk
    """

    tmppath = filepath + '~'
    while os.path.isfile(tmppath):
        tmppath += '~'
    try:
        with open(tmppath, 'wb' if binary else 'w') as file:
            yield file
            if fsync:
                file.flush()
                os.fsync(file.fileno())
        replace(tmppath, filepath)
    finally:
        try:
            os.remove(tmppath)
        except (IOError, OSError):
            pass 
Example 43
Project: qgis_resources_sharing   Author: akbargumbira   File: object_store.py    (license) View Source Project 6 votes vote down vote up
def add_pack(self):
        """Add a new pack to this object store.

        :return: Fileobject to write to, a commit function to
            call when the pack is finished and an abort
            function.
        """
        fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
        f = os.fdopen(fd, 'wb')
        def commit():
            os.fsync(fd)
            f.close()
            if os.path.getsize(path) > 0:
                return self.move_in_pack(path)
            else:
                os.remove(path)
                return None
        def abort():
            f.close()
            os.remove(path)
        return f, commit, abort 
Example 44
Project: HoneySMB   Author: r0hi7   File: smbserver.py    (license) View Source Project 6 votes vote down vote up
def smb2Flush(connId, smbServer, recvPacket):
        connData = smbServer.getConnectionData(connId)

        respSMBCommand = smb2.SMB2Flush_Response()
        flushRequest   = smb2.SMB2Flush(recvPacket['Data'])

        if connData['OpenedFiles'].has_key(str(flushRequest['FileID'])):
             fileHandle = connData['OpenedFiles'][str(flushRequest['FileID'])]['FileHandle']
             errorCode = STATUS_SUCCESS
             try:
                 os.fsync(fileHandle)
             except Exception, e:
                 smbServer.log("SMB2_FLUSH %s" % e, logging.ERROR)
                 errorCode = STATUS_ACCESS_DENIED
        else:
            errorCode = STATUS_INVALID_HANDLE

        smbServer.setConnectionData(connId, connData)
        return [respSMBCommand], None, errorCode 
Example 45
Project: enigma2-plugins   Author: opendreambox   File: plugin.py    (license) View Source Project 6 votes vote down vote up
def btnGreen(self):
		if self["list"].l.getCurrentSelection() is not None:
			list = self.list.getSelectionsList()
			if len(list) is not 0:
				for item in list:
					if "\""+item[1]+"\"" not in open("/etc/enigma2/bouquets.tv").read():
						os.system("cp /tmp/bouquet/"+item[1]+" /etc/enigma2/"+item[1])
						f = open("/etc/enigma2/bouquets.tv", 'a')
						f.write("#SERVICE 1:7:1:0:0:0:0:0:0:0:FROM BOUQUET \""+item[1]+"\" ORDER BY bouquet\n")
						f.flush()
						os.fsync(f.fileno())
						f.close()

				restart = self.session.openWithCallback(self.restart, MessageBox, "%s\n\n%s\n%s" %(_("selected bouquets are installed"), _("GUI needs a restart to apply changes."), _("Do you want to restart the GUI now?")), MessageBox.TYPE_YESNO)
				restart.setTitle("%s" %(_("Restart GUI now?")))

			else:
				self["info"].setText("%s" %(_("Please select at least one bouquet"))) 
Example 46
Project: charm-swift-proxy   Author: openstack   File: test_swift_context.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def test_get_swift_hash_file(self, mock_config):
        expected = '##FILEHASH##'
        with tempfile.NamedTemporaryFile() as tmpfile:
            swift_context.SWIFT_HASH_FILE = tmpfile.name
            tmpfile.write(expected.encode('UTF-8'))
            tmpfile.seek(0)
            os.fsync(tmpfile)
            hash = swift_context.get_swift_hash()

        self.assertFalse(mock_config.called)
        self.assertEqual(expected, hash) 
Example 47
Project: kinect-2-libras   Author: inessadl   File: mailbox.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def _sync_flush(f):
    """Ensure changes to file f are physically on disk."""
    f.flush()
    if hasattr(os, 'fsync'):
        os.fsync(f.fileno()) 
Example 48
Project: protofuzz   Author: trailofbits   File: log.py    (MIT License) View Source Project 5 votes vote down vote up
def _write_entries(self, entries):
        log_file = open(self._filename, 'wb')
        try:
            log_file.seek(0)
            for entry in entries:
                pickle.dump(entry, log_file, pickle.HIGHEST_PROTOCOL)
            log_file.flush()
            os.fsync(log_file.fileno())
        finally:
            log_file.close() 
Example 49
Project: kAFL   Author: RUB-SysSec   File: util.py    (GNU General Public License v2.0) View Source Project 5 votes vote down vote up
def atomic_write(filename, data):
    f = open("/tmp/kafl.tmp", 'wb')
    f.write(data)
    f.flush()
    os.fsync(f.fileno())
    f.close()
    os.rename("/tmp/kafl.tmp", filename) 
Example 50
Project: blobfs   Author: mbartoli   File: blobfs.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def flush(self, path, fh):
		print "flush:   " + path
		return os.fsync(fh)