Python io.BufferedReader() Examples

The following are 30 code examples of io.BufferedReader(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module io , or try the search function .
Example #1
Source File: umis.py    From umis with MIT License 13 votes vote down vote up
def read_fastq(filename):
    """
    return a stream of FASTQ entries, handling gzipped and empty files
    """
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3():
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh) 
Example #2
Source File: test_tinydb_reader.py    From sacred with MIT License 6 votes vote down vote up
def test_fetch_files_function(tmpdir):
    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name="experiment 1 alpha", exp_id="1234", root_dir=root)
    run_test_experiment(exp_name="experiment 2 beta", exp_id="5678", root_dir=root)
    run_test_experiment(
        exp_name="experiment 3 alpha beta", exp_id="9990", root_dir=root
    )

    tinydb_reader = TinyDbReader(root)

    res = tinydb_reader.fetch_files(indices=0)
    assert len(res) == 1
    assert list(res[0]["artifacts"].keys()) == ["about"]
    assert isinstance(res[0]["artifacts"]["about"], io.BufferedReader)
    assert res[0]["date"] == datetime.datetime(1999, 5, 4, 3, 2, 1)
    assert res[0]["exp_id"] == "1234"
    assert res[0]["exp_name"] == "experiment 1 alpha"
    assert list(res[0]["resources"].keys()) == ["sacred/__init__.py"]
    assert isinstance(res[0]["resources"]["sacred/__init__.py"], io.BufferedReader)
    assert list(res[0]["sources"].keys()) == ["setup.py"]
    assert isinstance(res[0]["sources"]["setup.py"], io.BufferedReader) 
Example #3
Source File: request.py    From pledgeservice with Apache License 2.0 6 votes vote down vote up
def _body_file__get(self):
        """
            Input stream of the request (wsgi.input).
            Setting this property resets the content_length and seekable flag
            (unlike setting req.body_file_raw).
        """
        if not self.is_body_readable:
            return io.BytesIO()
        r = self.body_file_raw
        clen = self.content_length
        if not self.is_body_seekable and clen is not None:
            # we need to wrap input in LimitedLengthFile
            # but we have to cache the instance as well
            # otherwise this would stop working
            # (.remaining counter would reset between calls):
            #   req.body_file.read(100)
            #   req.body_file.read(100)
            env = self.environ
            wrapped, raw = env.get('webob._body_file', (0,0))
            if raw is not r:
                wrapped = LimitedLengthFile(r, clen)
                wrapped = io.BufferedReader(wrapped)
                env['webob._body_file'] = wrapped, r
            r = wrapped
        return r 
Example #4
Source File: test_reactor_config.py    From atomic-reactor with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def test_invalid_schema_resource(self, tmpdir, caplog, schema):
        class FakeProvider(object):
            def get_resource_stream(self, pkg, rsc):
                return io.BufferedReader(io.BytesIO(schema))

        # pkg_resources.resource_stream() cannot be mocked directly
        # Instead mock the module-level function it calls.
        (flexmock(pkg_resources)
            .should_receive('get_provider')
            .and_return(FakeProvider()))

        filename = os.path.join(str(tmpdir), 'config.yaml')
        with open(filename, 'w'):
            pass

        tasker, workflow = self.prepare()
        plugin = ReactorConfigPlugin(tasker, workflow, config_path=str(tmpdir))
        with caplog.at_level(logging.ERROR), pytest.raises(Exception):
            plugin.run()

        captured_errs = [x.message for x in caplog.records]
        assert any("cannot validate" in x for x in captured_errs) 
Example #5
Source File: nbt.py    From nbtlib with MIT License 6 votes vote down vote up
def from_fileobj(cls, fileobj, byteorder="big"):
        """Load an nbt file from a proper file object.

        The method is used by the :func:`load` helper function when the
        ``gzipped`` keyword-only argument is not specified explicitly.

        Arguments:
            fileobj:
                Can be either a standard ``io.BufferedReader`` for
                uncompressed nbt or a ``gzip.GzipFile`` for gzipped nbt
                data. The function simply calls the inherited
                :meth:`nbtlib.tag.Compound.parse` classmethod and sets the
                :attr:`filename` and :attr:`gzipped` attributes depending
                on the argument.

            byteorder:
                Can be either ``"big"`` or ``"little"``. The argument is
                forwarded to :meth:`nbtlib.tag.Compound.parse`.
        """
        self = cls.parse(fileobj, byteorder)
        self.filename = getattr(fileobj, "name", self.filename)
        self.gzipped = isinstance(fileobj, gzip.GzipFile)
        self.byteorder = byteorder
        return self 
Example #6
Source File: in_memory_files_ops.py    From douglas-quaid with GNU General Public License v3.0 6 votes vote down vote up
def get_SHA1(file_value: io.BufferedReader):
    """
    Get SHA1 hash of the file, directly in memory
    TODO : Fix the input type
    :param file_value: A file to compute the SHA-1
    :return: the SHA-1 of the file in memory
    """

    h = hashlib.sha1()
    b = bytearray(128 * 1024)
    mv = memoryview(b)

    for n in iter(lambda: file_value.readinto(mv), 0):
        h.update(mv[:n])

    return h.hexdigest() 
Example #7
Source File: reckoner.py    From reckoner with Apache License 2.0 6 votes vote down vote up
def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False, create_namespace=True):
        self.config = Config()
        self.results = ReckonerInstallResults()
        self.config.dryrun = dryrun
        self.config.debug = debug
        self.config.helm_args = helm_args
        self.config.continue_on_error = continue_on_error
        self.config.create_namespace = create_namespace
        if course_file:
            self.config.course_path = course_file.name

        if self.config.debug:
            logging.warn("The --debug flag will be deprecated.  Please use --helm-args or --dry-run instead.")
        if self.config.helm_args:
            logging.warn("Specifying --helm-args on the cli will override helm_args in the course file.")

        self.course = Course(course_file) 
Example #8
Source File: indexing.py    From ms_deisotope with Apache License 2.0 6 votes vote down vote up
def idzip_compression(path, output):
        '''Compress a file using  idzip, a gzip-compatible format with random access support.
        '''
        if output is None:
            output = '-'
        with click.open_file(output, mode='wb') as outfh:
            writer = _compression.GzipFile(fileobj=outfh, mode='wb')
            with click.open_file(path, 'rb') as infh:
                try:
                    infh_wrap = io.BufferedReader(infh)
                    header = infh_wrap.peek(2)
                    if _compression.starts_with_gz_magic(header):
                        click.echo("Detected gzip input file", err=True)
                        infh_wrap = _compression.GzipFile(fileobj=infh_wrap)
                except AttributeError:
                    infh_wrap = infh
                buffer_size = _compression.WRITE_BUFFER_SIZE
                chunk = infh_wrap.read(buffer_size)
                while chunk:
                    writer.write(chunk)
                    chunk = infh_wrap.read(buffer_size)
            writer.close() 
Example #9
Source File: __init__.py    From aegea with Apache License 2.0 5 votes vote down vote up
def get_pricing_data(service_code, filters=None, max_cache_age_days=30):
    from ... import config

    if filters is None:
        filters = [("location", region_name(clients.ec2.meta.region_name))]

    get_products_args = dict(ServiceCode=service_code,
                             Filters=[dict(Type="TERM_MATCH", Field=k, Value=v) for k, v in filters])
    cache_key = hashlib.sha256(json.dumps(get_products_args, sort_keys=True).encode()).hexdigest()[:32]
    service_code_filename = os.path.join(config.user_config_dir, "pricing_cache_{}.json.gz".format(cache_key))
    try:
        cache_date = datetime.fromtimestamp(os.path.getmtime(service_code_filename))
        if cache_date < datetime.now() - timedelta(days=max_cache_age_days):
            raise Exception("Cache is too old, discard")
        with gzip.open(service_code_filename) as gz_fh:
            with io.BufferedReader(gz_fh) as buf_fh:
                pricing_data = json.loads(buf_fh.read().decode())
    except Exception:
        logger.info("Fetching pricing data for %s", service_code)
        client = boto3.client("pricing", region_name="us-east-1")
        pricing_data = [json.loads(p) for p in paginate(client.get_paginator("get_products"), **get_products_args)]
        try:
            with gzip.open(service_code_filename, "w") as fh:
                fh.write(json.dumps(pricing_data).encode())
        except Exception as e:
            print(e, file=sys.stderr)
    return pricing_data 
Example #10
Source File: test_smbclient_os.py    From smbprotocol with MIT License 5 votes vote down vote up
def test_read_byte_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = b"\x00\x01\x02\x03"

    expected = "[NtStatus 0xc0000034] No such file or directory"
    with pytest.raises(SMBOSError, match=re.escape(expected)):
        smbclient.open_file(file_path, mode='rb')

    with smbclient.open_file(file_path, mode='wb') as fd:
        fd.write(file_contents)

    with smbclient.open_file(file_path, mode='rb') as fd:
        assert isinstance(fd, io.BufferedReader)
        assert fd.closed is False
        assert fd.name == file_path

        actual = fd.read()
        assert actual == file_contents

        actual = fd.read()
        assert actual == b""

        fd.seek(0)
        actual = fd.read()
        assert actual == file_contents

        with pytest.raises(IOError):
            fd.write(b"Fail")
    assert fd.closed 
Example #11
Source File: tarfile.py    From jawfish with MIT License 5 votes vote down vote up
def extractfile(self, member):
        """Extract a member from the archive as a file object. `member' may be
           a filename or a TarInfo object. If `member' is a regular file or a
           link, an io.BufferedReader object is returned. Otherwise, None is
           returned.
        """
        self._check("r")

        if isinstance(member, str):
            tarinfo = self.getmember(member)
        else:
            tarinfo = member

        if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES:
            # Members with unknown types are treated as regular files.
            return self.fileobject(self, tarinfo)

        elif tarinfo.islnk() or tarinfo.issym():
            if isinstance(self.fileobj, _Stream):
                # A small but ugly workaround for the case that someone tries
                # to extract a (sym)link as a file-object from a non-seekable
                # stream of tar blocks.
                raise StreamError("cannot extract (sym)link as file object")
            else:
                # A (sym)link's file object is its target's file object.
                return self.extractfile(self._find_link_target(tarinfo))
        else:
            # If there's no data associated with the member (directory, chrdev,
            # blkdev, etc.), return None instead of a file object.
            return None 
Example #12
Source File: makefile.py    From gist-alfred with MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #13
Source File: makefile.py    From misp42splunk with GNU Lesser General Public License v3.0 5 votes vote down vote up
def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #14
Source File: makefile.py    From misp42splunk with GNU Lesser General Public License v3.0 5 votes vote down vote up
def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #15
Source File: ftp.py    From d6tpipe with MIT License 5 votes vote down vote up
def open(self, mode):
        """
        Open the FileSystem target.

        This method returns a file-like object which can either be read from or written to depending
        on the specified mode.

        :param mode: the mode `r` opens the FileSystemTarget in read-only mode, whereas `w` will
                     open the FileSystemTarget in write mode. Subclasses can implement
                     additional options.
        :type mode: str
        """
        if mode == 'w':
            return self.format.pipe_writer(AtomicFtpFile(self._fs, self.path))

        elif mode == 'r':
            temp_dir = os.path.join(tempfile.gettempdir(), 'luigi-contrib-ftp')
            self.__tmp_path = temp_dir + '/' + self.path.lstrip('/') + '-luigi-tmp-%09d' % random.randrange(0, 1e10)
            # download file to local
            self._fs.get(self.path, self.__tmp_path)

            return self.format.pipe_reader(
                FileWrapper(io.BufferedReader(io.FileIO(self.__tmp_path, 'r')))
            )
        else:
            raise Exception("mode must be 'r' or 'w' (got: %s)" % mode) 
Example #16
Source File: backport_makefile.py    From snowflake-connector-python with Apache License 2.0 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """Backport of socket.makefile from Python 3.5."""
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode {!r} (only r, w, b allowed)".format(mode)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #17
Source File: makefile.py    From core with MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #18
Source File: event_storage_reader.py    From thingsboard-gateway with Apache License 2.0 5 votes vote down vote up
def get_or_init_buffered_reader(self, pointer):
        try:
            if self.buffered_reader is None or self.buffered_reader.closed:
                new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file()
                # if not exists(new_file_to_read_path):
                #     next_file = self.get_next_file(self.files, self.new_pos)
                #     if next_file is not None:
                #         new_file_to_read_path = self.settings.get_data_folder_path() + next_file
                #     else:
                #         self.buffered_reader = None
                #         return None
                self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r'))
                lines_to_skip = pointer.get_line()
                if lines_to_skip > 0:
                    while self.buffered_reader.readline() is not None:
                        if lines_to_skip > 0:
                            lines_to_skip -= 1
                        else:
                            break

            return self.buffered_reader

        except IOError as e:
            log.error("Failed to initialize buffered reader! Error: %s", e)
            raise RuntimeError("Failed to initialize buffered reader!", e)
        except Exception as e:
            log.exception(e) 
Example #19
Source File: event_storage_reader.py    From thingsboard-gateway with Apache License 2.0 5 votes vote down vote up
def read_state_file(self):
        try:
            state_data_node = {}
            try:
                with BufferedReader(FileIO(self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as buffered_reader:
                    state_data_node = load(buffered_reader)
            except JSONDecodeError:
                log.error("Failed to decode JSON from state file")
                state_data_node = 0
            except IOError as e:
                log.warning("Failed to fetch info from state file! Error: %s", e)
            reader_file = None
            reader_pos = 0
            if state_data_node:
                reader_pos = state_data_node['position']
                for file in sorted(self.files.get_data_files()):
                    if file == state_data_node['file']:
                        reader_file = file
                        break
            if reader_file is None:
                reader_file = sorted(self.files.get_data_files())[0]
                reader_pos = 0
            log.info("FileStorage_reader -- Initializing from state file: [%s:%i]",
                     self.settings.get_data_folder_path() + reader_file,
                     reader_pos)
            return EventStorageReaderPointer(reader_file, reader_pos)
        except Exception as e:
            log.exception(e) 
Example #20
Source File: makefile.py    From ServerlessCrawler-VancouverRealState with MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #21
Source File: makefile.py    From ServerlessCrawler-VancouverRealState with MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #22
Source File: connections.py    From ServerlessCrawler-VancouverRealState with MIT License 5 votes vote down vote up
def _makefile(sock, mode):
        return io.BufferedReader(SocketIO(sock, mode)) 
Example #23
Source File: makefile.py    From ServerlessCrawler-VancouverRealState with MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #24
Source File: py3k.py    From recruit with Apache License 2.0 5 votes vote down vote up
def isfileobj(f):
        return isinstance(f, (io.FileIO, io.BufferedReader, io.BufferedWriter)) 
Example #25
Source File: _winconsole.py    From recruit with Apache License 2.0 5 votes vote down vote up
def _get_text_stdin(buffer_stream):
    text_stream = _NonClosingTextIOWrapper(
        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
        'utf-16-le', 'strict', line_buffering=True)
    return ConsoleStream(text_stream, buffer_stream) 
Example #26
Source File: _winconsole.py    From jbox with MIT License 5 votes vote down vote up
def _get_text_stdin(buffer_stream):
    text_stream = _NonClosingTextIOWrapper(
        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
        'utf-16-le', 'strict', line_buffering=True)
    return ConsoleStream(text_stream, buffer_stream) 
Example #27
Source File: umis.py    From umis with MIT License 5 votes vote down vote up
def read_cbhistogram(filename):
    if not filename:
        return None
    if filename.endswith('gz'):
#        filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
        filename_fh = gzip.open(filename, mode='rt')
    else:
        filename_fh = open(filename)
    return filename_fh 
Example #28
Source File: util_functions.py    From bale-bot-python with Apache License 2.0 5 votes vote down vote up
def get_file_buffer(file, file_operation_mode="rb"):
    if isinstance(file, str):
        opened_file = open(file, file_operation_mode)
        buffer = opened_file.read()
        opened_file.close()
    elif isinstance(file, BufferedReader):
        buffer = file.read()
        file.close()
    elif isinstance(file, bytes):
        buffer = file
    else:
        return None

    return buffer 
Example #29
Source File: makefile.py    From NEIE-Assistant with GNU General Public License v3.0 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= set(["r", "w", "b"]):
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example #30
Source File: cluster.py    From tattle with Mozilla Public License 2.0 5 votes vote down vote up
def _handle_udp_data(self, data, client_addr):
        try:

            # create a buffered reader
            reader = io.BufferedReader(io.BytesIO(data))

            # read until closed
            while True:

                # read a message
                try:
                    raw = self._read_udp_message(reader)
                    if raw is None:
                        break
                except IOError:
                    LOG.exception("Error reading stream")
                    break

                # decode the message
                msg = self._decode_message(raw)
                if msg is None:
                    continue

                # dispatch the message
                await self._handle_udp_message(msg, client_addr)

        except Exception:
            LOG.exception("Error handling UDP data")
            return

    # noinspection PyTypeChecker