Python io.FileIO() Examples

def __connect(self):
        device_addr, hidraw_device, event_device = self.__find_device()
        if device_addr is None:
            return False
        self.__report_fd = os.open(hidraw_device, os.O_RDWR | os.O_NONBLOCK)
        self.__fd = FileIO(self.__report_fd, "rb+", closefd=False)
        self.__input_device = InputDevice(event_device)
        self.__input_device.grab()
        buf = bytearray(38)
        buf[0] = 0x02
        try:
            return bool(fcntl.ioctl(self.__fd, 3223734279, bytes(buf)))
        except:
            pass
        if self.recv():
            self.update_controller() 

def __init__(self, logfile=None, verbose=True, debug=False, seed=None, tmp_dir=None):
		"""
			Anonymize fasta sequences

			@attention: 'shuf' is used which loads everything into memory!

			@param logfile: file handler or file path to a log file
			@type logfile: file | io.FileIO | StringIO.StringIO | str | unicode
			@param verbose: Not verbose means that only warnings and errors will be past to stream
			@type verbose: bool
			@param debug: more output and files are kept, manual clean up required
			@type debug: bool
			@param seed: The seed written to the random_source file used by the 'shuf' command
			@type seed: long | int | float | str | unicode
			@param tmp_dir: directory for temporary files, like the random_source file for 'shuf'
			@type tmp_dir: str | unicode

			@return: None
			@rtype: None
		"""
		assert isinstance(verbose, bool)
		assert isinstance(debug, bool)
		assert seed is None or isinstance(seed, (long, int, float, basestring))
		assert tmp_dir is None or isinstance(tmp_dir, basestring)
		if tmp_dir is not None:
			assert self.validate_dir(tmp_dir)
		else:
			tmp_dir = tempfile.gettempdir()
		self._tmp_dir = tmp_dir
		super(FastaAnonymizer, self).__init__(logfile, verbose, debug, label="FastaAnonymizer")

		if seed is not None:
			random.seed(seed)

		script_dir = os.path.dirname(self.get_full_path(__file__))
		self._anonymizer = os.path.join(script_dir, "anonymizer.py")
		self._fastastreamer = os.path.join(script_dir, "fastastreamer.py")
		assert self.validate_file(self._anonymizer)
		assert self.validate_file(self._fastastreamer) 

def add_log_stream(self, stream=sys.stderr, level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@param stream: stderr/stdout or a file stream
		@type stream: file | FileIO | StringIO
		@param level: minimum level of messages to be logged
		@type level: int | long

		@return: None
		@rtype: None
		"""
		assert self.is_stream(stream)
		# assert isinstance(stream, (file, io.FileIO))
		assert level in self._levelNames

		err_handler = logging.StreamHandler(stream)
		err_handler.setFormatter(self.message_formatter)
		err_handler.setLevel(level)
		self._logger.addHandler(err_handler) 

def __init__(self, default_compression="gz", logfile=None, verbose=True):
        """
        Constructor

        @attention:

        @param default_compression: default compression used for files
        @type default_compression: str | unicode
        @param logfile: file handler or file path to a log file
        @type logfile: file | io.FileIO | StringIO.StringIO | basestring
        @param verbose: Not verbose means that only warnings and errors will be past to stream
        @type verbose: bool

        @return: None
        @rtype: None
        """
        assert logfile is None or isinstance(logfile, basestring) or self.is_stream(logfile)
        assert isinstance(default_compression, basestring), "separator must be string"
        assert isinstance(verbose, bool), "verbose must be true or false"
        assert default_compression.lower() in self._open, "Unknown compression: '{}'".format(default_compression)

        super(Archive, self).__init__(label="Archive", default_compression=default_compression, logfile=logfile, verbose=verbose)

        self._open['tar'] = tarfile.open
        self._default_compression = default_compression 

def __init__(self, separator="\t", logfile=None, verbose=True):
		"""
			Handle tab separated files

			@attention:

			@param separator: default character assumed to separate values in a file
			@type separator: str | unicode
			@param logfile: file handler or file path to a log file
			@type logfile: file | io.FileIO | StringIO.StringIO | basestring
			@param verbose: Not verbose means that only warnings and errors will be past to stream
			@type verbose: bool

			@return: None
			@rtype: None
		"""
		assert logfile is None or isinstance(logfile, basestring) or self.is_stream(logfile)
		assert isinstance(separator, basestring), "separator must be string"
		assert isinstance(verbose, bool), "verbose must be true or false"
		super(MetadataTable, self).__init__(label="MetadataReader", logfile=logfile, verbose=verbose)

		self._number_of_rows = 0
		self._meta_table = {}
		self._separator = separator
		self._list_of_column_names = [] 

def add_log_stream(self, stream=sys.stderr, level=logging.INFO):
        """
        Add a stream where messages are outputted to.

        @param stream: stderr/stdout or a file stream
        @type stream: file | FileIO | StringIO
        @param level: minimum level of messages to be logged
        @type level: int | long

        @return: None
        @rtype: None
        """
        assert self.is_stream(stream)
        # assert isinstance(stream, (file, io.FileIO))
        assert level in self._levelNames

        err_handler = logging.StreamHandler(stream)
        err_handler.setFormatter(self.message_formatter)
        err_handler.setLevel(level)
        self._logger.addHandler(err_handler) 

def download_file(service, file_id, location, filename, mime_type):

    if 'vnd.google-apps' in mime_type:
        request = service.files().export_media(fileId=file_id,
                mimeType='application/pdf')
        filename += '.pdf'
    else:
        request = service.files().get_media(fileId=file_id)
    fh = io.FileIO(location + filename, 'wb')
    downloader = MediaIoBaseDownload(fh, request, 1024 * 1024 * 1024)
    done = False
    while done is False:
        try:
            status, done = downloader.next_chunk()
        except:
            fh.close()
            os.remove(location + filename)
            sys.exit(1)
        print(f'\rDownload {int(status.progress() * 100)}%.', end='')
        sys.stdout.flush()
    print('') 

def _get_cbar_png_bytes(cmap):
    gradient = np.linspace(0, 1, 256)
    gradient = np.vstack((gradient, gradient))
    image_data = cmap(gradient, bytes=True)
    image = Image.fromarray(image_data, 'RGBA')

    # ostream = io.FileIO('../cmaps/' + cmap_name + '.png', 'wb')
    # image.save(ostream, format='PNG')
    # ostream.close()

    ostream = io.BytesIO()
    image.save(ostream, format='PNG')
    cbar_png_bytes = ostream.getvalue()
    ostream.close()

    cbar_png_data = base64.b64encode(cbar_png_bytes)
    cbar_png_bytes = cbar_png_data.decode('unicode_escape')

    return cbar_png_bytes 

def _log_message(self, path: str, message: str) -> None:
        # We use io.FileIO here because it guarantees that write() is implemented with a single write syscall,
        # and on Linux, writes to O_APPEND files with a single write syscall are atomic.
        #
        # https://docs.python.org/2/library/io.html#io.FileIO
        # http://article.gmane.org/gmane.linux.kernel/43445

        try:
            with io.FileIO(path, mode=self.mode, closefd=True) as f:
                with self.maybe_flock(f):
                    f.write(message.encode("UTF-8"))
        except IOError as e:
            print(
                "Could not log to {}: {}: {} -- would have logged: {}".format(
                    path, type(e).__name__, str(e), message
                ),
                file=sys.stderr,
            ) 

def create_manifest(data_path, tag, ordered=True):
    manifest_path = '%s_manifest.csv' % tag
    file_paths = []
    wav_files = [os.path.join(dirpath, f)
                 for dirpath, dirnames, files in os.walk(data_path)
                 for f in fnmatch.filter(files, '*.wav')]
    size = len(wav_files)
    counter = 0
    for file_path in wav_files:
        file_paths.append(file_path.strip())
        counter += 1
        update_progress(counter / float(size))
    print('\n')
    if ordered:
        _order_files(file_paths)
    counter = 0
    with io.FileIO(manifest_path, "w") as f:
        for wav_path in file_paths:
            transcript_path = wav_path.replace('/wav/', '/txt/').replace('.wav', '.txt')
            sample = os.path.abspath(wav_path) + ',' + os.path.abspath(transcript_path) + '\n'
            f.write(sample.encode('utf-8'))
            counter += 1
            update_progress(counter / float(size))
    print('\n') 

def stream_directory(self, directory, out_stream=sys.stdout, file_format="fastq", extension="fq", paired=False):
		"""
			Stream sequences consecutively

			@attention:

			@param directory: A directory
			@type directory: basestring
			@param out_stream: A stream the output will be written to.
			@type out_stream: file | io.FileIO | StringIO.StringIO
			@param file_format: Fasta format of input and output. Either 'fasta' or 'fastq'.
			@type file_format: basestring
			@param extension: file extension to be filtered for
			@type extension: basestring
			@param paired: sequences are streamed interweaved from a pair of files if True, else consecutively
			@type paired: bool

			@return: None
			@rtype: None
		"""
		assert isinstance(directory, basestring)
		directory = FastaStreamer.get_full_path(directory)
		assert self.validate_dir(directory)
		assert self.is_stream(out_stream)
		assert isinstance(file_format, basestring)
		file_format = file_format.lower()
		assert file_format in self._legal_formats
		assert extension is None or isinstance(extension, basestring)

		list_of_file = self.get_files_in_directory(directory, extension=extension)
		if not paired:
			self.consecutive_stream(list_of_file, out_stream=out_stream, file_format=file_format)
		else:
			self.interweave_stream(list_of_file, out_stream=out_stream, file_format=file_format, extension=extension) 

def stream_file(self, file_path, out_stream=sys.stdout, file_format="fastq", paired=False):
		"""
			Stream sequences consecutively

			@attention:

			@param file_path: A file path
			@type file_path: basestring
			@param out_stream: A stream the output will be written to.
			@type out_stream: file | io.FileIO | StringIO.StringIO
			@param file_format: Fasta format of input and output. Either 'fasta' or 'fastq'.
			@type file_format: basestring
			@param paired: sequences are streamed as pair, else one by one
			@type paired: bool

			@return: None
			@rtype: None
		"""
		assert isinstance(file_path, basestring)
		file_path = FastaStreamer.get_full_path(file_path)
		assert self.validate_file(file_path)
		assert self.is_stream(out_stream)
		assert isinstance(file_format, basestring)
		file_format = file_format.lower()
		assert file_format in self._legal_formats

		self.consecutive_stream(file_path, out_stream=out_stream, file_format=file_format, paired=paired) 

def is_stream(stream):
		return isinstance(stream, (file, io.FileIO, StringIO.StringIO)) or stream.__class__ is StringIO.StringIO 

def set_log_file(self, log_file, mode='a', level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@attention: file stream will only be closed if a file path is given!

		@param log_file: file stream or file path of logfile
		@type log_file: file | FileIO | StringIO | basestring
		@param mode: opening mode for logfile, if a file path is given
		@type mode: basestring
		@param level: minimum level of messages to be logged
		@type level: int or long

		@return: None
		@rtype: None
		"""
		assert isinstance(log_file, basestring) or self.is_stream(log_file)
		assert level in self._levelNames

		if LoggingWrapper._map_logfile_handler[self._label] is not None:
			self._logger.removeHandler(LoggingWrapper._map_logfile_handler[self._label])
			LoggingWrapper._map_logfile_handler[self._label].close()
			LoggingWrapper._map_logfile_handler[self._label] = None

		if self.is_stream(log_file):
			self.add_log_stream(stream=log_file, level=level)
			return

		try:
			err_handler_file = logging.FileHandler(log_file, mode)
			err_handler_file.setFormatter(self.message_formatter)
			err_handler_file.setLevel(level)
			self._logger.addHandler(err_handler_file)
			LoggingWrapper._map_logfile_handler[self._label] = err_handler_file
		except Exception:
			sys.stderr.write("[LoggingWrapper] Could not open '{}' for logging\n".format(log_file))
			return 

def is_stream(stream):
		"""
		Test for streams

		@param stream: Any kind of stream type
		@type stream: file | io.FileIO | StringIO.StringIO

		@return: True if stream
		@rtype: bool
		"""
		return isinstance(stream, (file, io.FileIO, StringIO.StringIO)) or stream.__class__ is StringIO.StringIO 

def __init__(self, default_compression="gz", label="Compress", logfile=None, verbose=True, debug=False):
        """
        Constructor

        @attention:

        @param default_compression: default compression used for files
        @type default_compression: str | unicode
        @param logfile: file handler or file path to a log file
        @type logfile: file | io.FileIO | StringIO.StringIO | basestring
        @param verbose: Not verbose means that only warnings and errors will be past to stream
        @type verbose: bool
        @param debug: Display debug messages
        @type debug: bool

        @return: None
        @rtype: None
        """
        assert logfile is None or isinstance(logfile, basestring) or self.is_stream(logfile)
        assert isinstance(default_compression, basestring), "separator must be string"
        assert isinstance(verbose, bool), "verbose must be true or false"
        assert default_compression.lower() in self._open, "Unknown compression: '{}'".format(default_compression)

        super(Compress, self).__init__(label=label, logfile=logfile, verbose=verbose, debug=debug)

        # self._logger = LoggingWrapper(self._label, verbose=verbose)
        # if logfile is not None:
        #     self._logger.set_log_file(logfile)

        self._default_compression = default_compression 

def _validate_file_start(self, stream_handle, file_format):
		"""
			Validate that a stream with sequences starts with the correct character

			@param stream_handle: Any kind of stream type
			@type stream_handle: file | io.FileIO | StringIO.StringIO
			@param file_format: Format of the file at the file_path provided. Valid: 'fasta', 'fastq'
			@type file_format: str | unicode

			@return: True if the first character is correct
			@rtype: bool
		"""
		assert self.is_stream(stream_handle)
		assert isinstance(file_format, basestring)
		file_format = file_format.lower()
		assert file_format in self._formats

		sequence_indicator = self._sequence_indicators[file_format]

		head = stream_handle.read(1)
		stream_handle.seek(0)
		if not head:
			return False
		if not head.startswith(sequence_indicator):
			return False
		return True 

def is_stream(stream):
        """
        Test for streams

        @param stream: Any kind of stream type
        @type stream: file | io.FileIO | StringIO.StringIO

        @return: True if stream
        @rtype: bool
        """
        return hasattr(stream, 'read') and hasattr(stream, 'write') 

def set_log_file(self, log_file, mode='a', level=logging.INFO):
        """
        Add a stream where messages are outputted to.

        @attention: file stream will only be closed if a file path is given!

        @param log_file: file stream or file path of logfile
        @type log_file: file | FileIO | StringIO | str
        @param mode: opening mode for logfile, if a file path is given
        @type mode: str
        @param level: minimum level of messages to be logged
        @type level: int or long

        @return: None
        @rtype: None
        """
        assert isinstance(log_file, str) or self.is_stream(log_file)
        assert level in self._levelNames

        if LoggingWrapper._map_logfile_handler[self._label] is not None:
            self._logger.removeHandler(LoggingWrapper._map_logfile_handler[self._label])
            LoggingWrapper._map_logfile_handler[self._label].close()
            LoggingWrapper._map_logfile_handler[self._label] = None

        if self.is_stream(log_file):
            self.add_log_stream(stream=log_file, level=level)
            return

        try:
            err_handler_file = logging.FileHandler(log_file, mode)
            err_handler_file.setFormatter(self.message_formatter)
            err_handler_file.setLevel(level)
            self._logger.addHandler(err_handler_file)
            LoggingWrapper._map_logfile_handler[self._label] = err_handler_file
        except Exception:
            sys.stderr.write("[LoggingWrapper] Could not open '{}' for logging\n".format(log_file))
            return 

def __init__(self, label="DefaultLogging", logfile=None, verbose=False, debug=False):
        """
        Prototype class for any class needing a logger

        @attention:

        @param logfile: file handler or file path to a log file
        @type logfile: file | FileIO | StringIO | str
        @param verbose: Not verbose means that only warnings and errors will be past to stream
        @type verbose: bool
        @param debug: Display debug messages
        @type debug: bool

        @return: None
        @rtype: None
        """
        assert isinstance(debug, bool)

        self._logger = LoggingWrapper(label, verbose=verbose)
        if logfile:
            self._logger.set_log_file(logfile, mode='a')

        self._debug = debug
        if debug:
            self._logger.set_level(self._logger.DEBUG)

        self._logfile = None
        if isinstance(logfile, str):
            self._logfile = logfile
        else:
            if hasattr(logfile, 'name'):
                self._logfile = logfile.name
        self._verbose = verbose 

def is_stream(stream):
        """
        Test for streams

        @param stream: Any kind of stream type
        @type stream: file | io.FileIO | StringIO.StringIO

        @return: True if stream
        @rtype: bool
        """
        return hasattr(stream, 'read') and hasattr(stream, 'write') 

def download(url, file_name):
    """
    function to download file over http
    url : URL of file to be downloaded
    file_name : File name
    """
    with io.FileIO(file_name, 'w') as file:
        # get request
        response = get(url)
        # write to file
        file.write(response.content) 

def chunked_copy(name, source, dest):
	"""
	Copies one stream into another, with progress bar.

	:param name: Name of the file to display.
	:param source: Source stream.
	:param dest: Destination stream.

	:return: Number of bytes copied.
	"""

	global is_conemu

	size = int(source.info()['Content-Length'].strip())
	recv = 0

	if len(name) > 23:
		name = name[0:20] + '...'

	hide_cursor()

	while True:
		chunk = source.read(8192)
		recv += len(chunk)

		if not chunk:
			break

		dest.write(chunk)

		draw_progress(recv, size, name)

	show_cursor()

	return recv


# FileIO wrapper with progress bar 

def __init__(self, path, *args, **kwargs):
		self._total_size = os.path.getsize(path)
		self.current_extraction = ''
		io.FileIO.__init__(self, path, *args, **kwargs)

		hide_cursor() 

def read(self, length):
		"""
		Read at most size bytes, returned as bytes.

		Only makes one system call, so less data may be returned than requested.
		In non-blocking mode, returns None if no data is available.
		Return an empty bytes object at EOF.
		"""

		draw_progress(self.tell(), self._total_size, self.current_extraction)

		return io.FileIO.read(self, length) 

def isfileobj(f):
        return isinstance(f, io.FileIO) 

def download_file(service, file_id, location, filename, mime_type):

    if 'vnd.google-apps' in mime_type:
        request = service.files().export_media(fileId=file_id,
                mimeType='application/pdf')
        filename += '.pdf'
    else:
        request = service.files().get_media(fileId=file_id)
    fh = io.FileIO(location + filename, 'wb')
    downloader = MediaIoBaseDownload(fh, request, 1024 * 1024 * 1024)
    done = False
    while done is False:
        try:
            status, done = downloader.next_chunk()
        except:
            fh.close()
            os.remove(location + filename)
            sys.exit(1)
        print '\rDownload {}%.'.format(int(status.progress() * 100)),
        sys.stdout.flush()
    print '' 

def get_or_init_buffered_reader(self, pointer):
        try:
            if self.buffered_reader is None or self.buffered_reader.closed:
                new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file()
                # if not exists(new_file_to_read_path):
                #     next_file = self.get_next_file(self.files, self.new_pos)
                #     if next_file is not None:
                #         new_file_to_read_path = self.settings.get_data_folder_path() + next_file
                #     else:
                #         self.buffered_reader = None
                #         return None
                self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r'))
                lines_to_skip = pointer.get_line()
                if lines_to_skip > 0:
                    while self.buffered_reader.readline() is not None:
                        if lines_to_skip > 0:
                            lines_to_skip -= 1
                        else:
                            break

            return self.buffered_reader

        except IOError as e:
            log.error("Failed to initialize buffered reader! Error: %s", e)
            raise RuntimeError("Failed to initialize buffered reader!", e)
        except Exception as e:
            log.exception(e) 

def read_state_file(self):
        try:
            state_data_node = {}
            try:
                with BufferedReader(FileIO(self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as buffered_reader:
                    state_data_node = load(buffered_reader)
            except JSONDecodeError:
                log.error("Failed to decode JSON from state file")
                state_data_node = 0
            except IOError as e:
                log.warning("Failed to fetch info from state file! Error: %s", e)
            reader_file = None
            reader_pos = 0
            if state_data_node:
                reader_pos = state_data_node['position']
                for file in sorted(self.files.get_data_files()):
                    if file == state_data_node['file']:
                        reader_file = file
                        break
            if reader_file is None:
                reader_file = sorted(self.files.get_data_files())[0]
                reader_pos = 0
            log.info("FileStorage_reader -- Initializing from state file: [%s:%i]",
                     self.settings.get_data_folder_path() + reader_file,
                     reader_pos)
            return EventStorageReaderPointer(reader_file, reader_pos)
        except Exception as e:
            log.exception(e) 

def get_or_init_buffered_writer(self, file):
        try:
            if self.buffered_writer is None or self.buffered_writer.closed:
                self.buffered_writer = BufferedWriter(FileIO(self.settings.get_data_folder_path() + file, 'a'))
            return self.buffered_writer
        except IOError as e:
            log.error("Failed to initialize buffered writer! Error: %s", e)
            raise RuntimeError("Failed to initialize buffered writer!", e)