Python io.FileIO() Examples

def __connect(self):
        device_addr, hidraw_device, event_device = self.__find_device()
        if device_addr is None:
            return False
        self.__report_fd = os.open(hidraw_device, os.O_RDWR | os.O_NONBLOCK)
        self.__fd = FileIO(self.__report_fd, "rb+", closefd=False)
        self.__input_device = InputDevice(event_device)
        self.__input_device.grab()
        buf = bytearray(38)
        buf[0] = 0x02
        try:
            return bool(fcntl.ioctl(self.__fd, 3223734279, bytes(buf)))
        except:
            pass
        if self.recv():
            self.update_controller() 

def __init__(self, logfile=None, verbose=True, debug=False, seed=None, tmp_dir=None):
		"""
			Anonymize fasta sequences

			@attention: 'shuf' is used which loads everything into memory!

			@param logfile: file handler or file path to a log file
			@type logfile: file | io.FileIO | StringIO.StringIO | str | unicode
			@param verbose: Not verbose means that only warnings and errors will be past to stream
			@type verbose: bool
			@param debug: more output and files are kept, manual clean up required
			@type debug: bool
			@param seed: The seed written to the random_source file used by the 'shuf' command
			@type seed: long | int | float | str | unicode
			@param tmp_dir: directory for temporary files, like the random_source file for 'shuf'
			@type tmp_dir: str | unicode

			@return: None
			@rtype: None
		"""
		assert isinstance(verbose, bool)
		assert isinstance(debug, bool)
		assert seed is None or isinstance(seed, (long, int, float, basestring))
		assert tmp_dir is None or isinstance(tmp_dir, basestring)
		if tmp_dir is not None:
			assert self.validate_dir(tmp_dir)
		else:
			tmp_dir = tempfile.gettempdir()
		self._tmp_dir = tmp_dir
		super(FastaAnonymizer, self).__init__(logfile, verbose, debug, label="FastaAnonymizer")

		if seed is not None:
			random.seed(seed)

		script_dir = os.path.dirname(self.get_full_path(__file__))
		self._anonymizer = os.path.join(script_dir, "anonymizer.py")
		self._fastastreamer = os.path.join(script_dir, "fastastreamer.py")
		assert self.validate_file(self._anonymizer)
		assert self.validate_file(self._fastastreamer) 

def add_log_stream(self, stream=sys.stderr, level=logging.INFO):
        """
        Add a stream where messages are outputted to.

        @param stream: stderr/stdout or a file stream
        @type stream: file | FileIO | StringIO
        @param level: minimum level of messages to be logged
        @type level: int | long

        @return: None
        @rtype: None
        """
        assert self.is_stream(stream)
        # assert isinstance(stream, (file, io.FileIO))
        assert level in self._levelNames

        err_handler = logging.StreamHandler(stream)
        err_handler.setFormatter(self.message_formatter)
        err_handler.setLevel(level)
        self._logger.addHandler(err_handler) 

def _get_cbar_png_bytes(cmap):
    gradient = np.linspace(0, 1, 256)
    gradient = np.vstack((gradient, gradient))
    image_data = cmap(gradient, bytes=True)
    image = Image.fromarray(image_data, 'RGBA')

    # ostream = io.FileIO('../cmaps/' + cmap_name + '.png', 'wb')
    # image.save(ostream, format='PNG')
    # ostream.close()

    ostream = io.BytesIO()
    image.save(ostream, format='PNG')
    cbar_png_bytes = ostream.getvalue()
    ostream.close()

    cbar_png_data = base64.b64encode(cbar_png_bytes)
    cbar_png_bytes = cbar_png_data.decode('unicode_escape')

    return cbar_png_bytes 

def __init__(self, separator="\t", logfile=None, verbose=True):
		"""
			Handle tab separated files

			@attention:

			@param separator: default character assumed to separate values in a file
			@type separator: str | unicode
			@param logfile: file handler or file path to a log file
			@type logfile: file | io.FileIO | StringIO.StringIO | basestring
			@param verbose: Not verbose means that only warnings and errors will be past to stream
			@type verbose: bool

			@return: None
			@rtype: None
		"""
		assert logfile is None or isinstance(logfile, basestring) or self.is_stream(logfile)
		assert isinstance(separator, basestring), "separator must be string"
		assert isinstance(verbose, bool), "verbose must be true or false"
		super(MetadataTable, self).__init__(label="MetadataReader", logfile=logfile, verbose=verbose)

		self._number_of_rows = 0
		self._meta_table = {}
		self._separator = separator
		self._list_of_column_names = [] 

def download_file(service, file_id, location, filename, mime_type):

    if 'vnd.google-apps' in mime_type:
        request = service.files().export_media(fileId=file_id,
                mimeType='application/pdf')
        filename += '.pdf'
    else:
        request = service.files().get_media(fileId=file_id)
    fh = io.FileIO(location + filename, 'wb')
    downloader = MediaIoBaseDownload(fh, request, 1024 * 1024 * 1024)
    done = False
    while done is False:
        try:
            status, done = downloader.next_chunk()
        except:
            fh.close()
            os.remove(location + filename)
            sys.exit(1)
        print(f'\rDownload {int(status.progress() * 100)}%.', end='')
        sys.stdout.flush()
    print('') 

def add_log_stream(self, stream=sys.stderr, level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@param stream: stderr/stdout or a file stream
		@type stream: file | FileIO | StringIO
		@param level: minimum level of messages to be logged
		@type level: int | long

		@return: None
		@rtype: None
		"""
		assert self.is_stream(stream)
		# assert isinstance(stream, (file, io.FileIO))
		assert level in self._levelNames

		err_handler = logging.StreamHandler(stream)
		err_handler.setFormatter(self.message_formatter)
		err_handler.setLevel(level)
		self._logger.addHandler(err_handler) 

def create_manifest(data_path, tag, ordered=True):
    manifest_path = '%s_manifest.csv' % tag
    file_paths = []
    wav_files = [os.path.join(dirpath, f)
                 for dirpath, dirnames, files in os.walk(data_path)
                 for f in fnmatch.filter(files, '*.wav')]
    size = len(wav_files)
    counter = 0
    for file_path in wav_files:
        file_paths.append(file_path.strip())
        counter += 1
        update_progress(counter / float(size))
    print('\n')
    if ordered:
        _order_files(file_paths)
    counter = 0
    with io.FileIO(manifest_path, "w") as f:
        for wav_path in file_paths:
            transcript_path = wav_path.replace('/wav/', '/txt/').replace('.wav', '.txt')
            sample = os.path.abspath(wav_path) + ',' + os.path.abspath(transcript_path) + '\n'
            f.write(sample.encode('utf-8'))
            counter += 1
            update_progress(counter / float(size))
    print('\n') 

def _log_message(self, path: str, message: str) -> None:
        # We use io.FileIO here because it guarantees that write() is implemented with a single write syscall,
        # and on Linux, writes to O_APPEND files with a single write syscall are atomic.
        #
        # https://docs.python.org/2/library/io.html#io.FileIO
        # http://article.gmane.org/gmane.linux.kernel/43445

        try:
            with io.FileIO(path, mode=self.mode, closefd=True) as f:
                with self.maybe_flock(f):
                    f.write(message.encode("UTF-8"))
        except IOError as e:
            print(
                "Could not log to {}: {}: {} -- would have logged: {}".format(
                    path, type(e).__name__, str(e), message
                ),
                file=sys.stderr,
            ) 

def __init__(self, default_compression="gz", logfile=None, verbose=True):
        """
        Constructor

        @attention:

        @param default_compression: default compression used for files
        @type default_compression: str | unicode
        @param logfile: file handler or file path to a log file
        @type logfile: file | io.FileIO | StringIO.StringIO | basestring
        @param verbose: Not verbose means that only warnings and errors will be past to stream
        @type verbose: bool

        @return: None
        @rtype: None
        """
        assert logfile is None or isinstance(logfile, basestring) or self.is_stream(logfile)
        assert isinstance(default_compression, basestring), "separator must be string"
        assert isinstance(verbose, bool), "verbose must be true or false"
        assert default_compression.lower() in self._open, "Unknown compression: '{}'".format(default_compression)

        super(Archive, self).__init__(label="Archive", default_compression=default_compression, logfile=logfile, verbose=verbose)

        self._open['tar'] = tarfile.open
        self._default_compression = default_compression 

def isfileobj(f):
        return isinstance(f, (io.FileIO, io.BufferedReader, io.BufferedWriter)) 

def test_tee_readable_readline(tmpdir):
    tmp = tmpdir.join('tmp')
    with FileIO(tmp.strpath, 'w') as fp:
        reader = TeeReadable(BytesIO(b'abc\n'), fp)
        assert reader.readline(10), b'abc\n' 

def load_from_file(filename):
        f = FileIO(filename, 'rb')
        data = f.readall()
        return ByteArray(data) 

def create_manifest(data_path, output_path, min_duration=None, max_duration=None):
    file_paths = [os.path.join(dirpath, f)
                  for dirpath, dirnames, files in os.walk(data_path)
                  for f in fnmatch.filter(files, '*.wav')]
    file_paths = order_and_prune_files(file_paths, min_duration, max_duration)
    with io.FileIO(output_path, "w") as file:
        for wav_path in tqdm(file_paths, total=len(file_paths)):
            transcript_path = wav_path.replace('/wav/', '/txt/').replace('.wav', '.txt')
            sample = os.path.abspath(wav_path) + ',' + os.path.abspath(transcript_path) + '\n'
            file.write(sample.encode('utf-8'))
    print('\n') 

def readables_from_file_path(path):
    """ Yield readables from a file system path """

    numdirs = 0
    for dirpath, dirnames, filenames in os.walk(path):
        numdirs += 1
        # Don't walk into "hidden" directories
        dirnames[:] = [n for n in dirnames if not n.startswith('.')]
        for filename in filenames:
            if filename.lower().endswith(EXT_WEXIN):
                filepath = os.path.join(dirpath, filename)
                yield Open(partial(FileIO, filepath))

    if numdirs < 1:
        if path.lower().endswith('EXT_WEXIN'):
            yield Open(partial(FileIO, path))
        else:
            try:
                tf = tarfile_open(path)
                for readable in readables_from_tarfile(tf):
                    yield readable
            except IOError as exc:
                if exc.errno != errno.ENOENT:
                    raise
                # we want to defer the ENOENT error to later
                # so we do that by yielding an Open(...)
                yield Open(partial(FileIO, path))
            except tarfile.ReadError:
                yield Open(partial(FileIO, path)) 

def test_tee_readable(tmpdir):
    tmp = tmpdir.join('tmp')
    with FileIO(tmp.strpath, 'w') as fp:
        reader = TeeReadable(BytesIO(b'abc'), fp)
        assert reader.read(10), b'abc'
        assert reader.name == tmp.strpath 

def test_pytest_collect_file(tmpdir, parent):
    # FTM just to see how to coverage test the plugin
    r0_wexin = tmpdir.join('0' + EXT_WEXIN)
    r0_wexout = tmpdir.join('0' + EXT_WEXOUT)
    with FileIO(r0_wexin.strpath, 'w') as fp:
        fp.write(response)
    with FileIO(r0_wexout.strpath, 'w') as fp:
        fp.write(b'this\t"that"\n')
    fileobj = pytestplugin.pytest_collect_file(parent, r0_wexin)
    item = next(fileobj.collect())
    item.runtest() 

def _is_raw_file(fileobj):
    """Check if fileobj is a raw file object, e.g created with open."""
    if PY3_OR_LATER:
        fileobj = getattr(fileobj, 'raw', fileobj)
        return isinstance(fileobj, io.FileIO)
    else:
        return isinstance(fileobj, file)  # noqa


###############################################################################
# Cache file utilities 

def isfileobj(f):
        return isinstance(f, (io.FileIO, io.BufferedReader)) 

def isfile(f):
        if isinstance(f, io.FileIO):
            return True
        elif hasattr(f, 'buffer'):
            return isfile(f.buffer)
        elif hasattr(f, 'raw'):
            return isfile(f.raw)
        return False 

def isfileobj(f):
        return isinstance(f, io.FileIO) 

def create_dummy_files_in_folder(temp_folder, file_format = 'txt'):
    create_folder(temp_folder)
    for ct in range(10):
        file_name = str(ct) + '.' + file_format
        file_path = os.path.join(temp_folder, file_name)
        with io.FileIO(file_path, "w") as file:
            file.write("Hello!") 

def __init__(self, fd: int, *args: Any, **kwargs: Any) -> None:
        self.fd = fd
        self._fio = io.FileIO(self.fd, "r+")
        _set_nonblocking(fd)
        super(PipeIOStream, self).__init__(*args, **kwargs) 

def __init__(self, fd: int, *args: Any, **kwargs: Any) -> None:
        self.fd = fd
        self._fio = io.FileIO(self.fd, "r+")
        _set_nonblocking(fd)
        super(PipeIOStream, self).__init__(*args, **kwargs) 

def __init__(self, fd, *args, **kwargs):
        self.fd = fd
        self._fio = io.FileIO(self.fd, "r+")
        _set_nonblocking(fd)
        super(PipeIOStream, self).__init__(*args, **kwargs) 

def isfileobj(f):
        return isinstance(f, (io.FileIO, io.BufferedReader, io.BufferedWriter)) 

def isfileobj(f):
        return isinstance(f, (io.FileIO, io.BufferedReader)) 

def unix_print_tty(string='', indents=0, newline=True):
    with contextlib2.ExitStack() as stack:
        string = indent(indents) + string
        fd = None

        try:
            # Always try reading and writing directly on the tty first.
            fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY)
            tty = io.FileIO(fd, 'w+')
            stack.enter_context(tty)
            text_input = io.TextIOWrapper(tty)
            stack.enter_context(text_input)
            stream = text_input
        except OSError:
            sys.stdout.write(string)

            if newline:
                sys.stdout.write('\n')

            stack.close()

        if fd is not None:
            try:
                stream.write(string)

                if newline:
                    stream.write('\n')
            finally:
                stream.flush() 

def download_file(service, file_id, location, filename):
    request = service.files().get_media(fileId=file_id)
    fh = io.FileIO('{}{}'.format(location, filename), 'wb')
    downloader = MediaIoBaseDownload(fh, request,chunksize=1024*1024)
    done = False
    while done is False:
        status, done = downloader.next_chunk()
        if status:
            #print '\rDownload {}%.'.format(int(status.progress() * 100)),
            print int(status.progress() * 100)," percent complete         \r",
            #sys.stdout.flush()
    print ""
    print colored(('%s downloaded!' % filename), 'green') 

def stream_directory(self, directory, out_stream=sys.stdout, file_format="fastq", extension="fq", paired=False):
		"""
			Stream sequences consecutively

			@attention:

			@param directory: A directory
			@type directory: basestring
			@param out_stream: A stream the output will be written to.
			@type out_stream: file | io.FileIO | StringIO.StringIO
			@param file_format: Fasta format of input and output. Either 'fasta' or 'fastq'.
			@type file_format: basestring
			@param extension: file extension to be filtered for
			@type extension: basestring
			@param paired: sequences are streamed interweaved from a pair of files if True, else consecutively
			@type paired: bool

			@return: None
			@rtype: None
		"""
		assert isinstance(directory, basestring)
		directory = FastaStreamer.get_full_path(directory)
		assert self.validate_dir(directory)
		assert self.is_stream(out_stream)
		assert isinstance(file_format, basestring)
		file_format = file_format.lower()
		assert file_format in self._legal_formats
		assert extension is None or isinstance(extension, basestring)

		list_of_file = self.get_files_in_directory(directory, extension=extension)
		if not paired:
			self.consecutive_stream(list_of_file, out_stream=out_stream, file_format=file_format)
		else:
			self.interweave_stream(list_of_file, out_stream=out_stream, file_format=file_format, extension=extension)