Python io.TextIOWrapper() Examples

def _fopen(fname, mode):
    """
    Extend file open function, to support 
        1) "-", which means stdin/stdout
        2) "$cmd |" which means pipe.stdout
    """
    if mode not in ["w", "r", "wb", "rb"]:
        raise ValueError("Unknown open mode: {mode}".format(mode=mode))
    if not fname:
        return None
    fname = fname.rstrip()
    if fname == "-":
        if mode in ["w", "wb"]:
            return sys.stdout.buffer if mode == "wb" else sys.stdout
        else:
            return sys.stdin.buffer if mode == "rb" else sys.stdin
    elif fname[-1] == "|":
        pin = pipe_fopen(fname[:-1], mode, background=(mode == "rb"))
        return pin if mode == "rb" else TextIOWrapper(pin)
    else:
        if mode in ["r", "rb"] and not os.path.exists(fname):
            raise FileNotFoundError(
                "Could not find common file: {}".format(fname))
        return open(fname, mode) 

def test_write_exclusive_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    with smbclient.open_file(file_path, mode='x') as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False

        with pytest.raises(IOError):
            fd.read()

        assert fd.tell() == 0
        fd.write(file_contents)
        assert int(fd.tell()) == (len(file_contents) - 1 + len(os.linesep))

    assert fd.closed is True

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == file_contents

    with pytest.raises(OSError, match=re.escape("[NtStatus 0xc0000035] File exists: ")):
        smbclient.open_file(file_path, mode='x')

    assert fd.closed is True 

def test_append_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")

    with smbclient.open_file(file_path, mode='a') as fd:
        assert isinstance(fd, io.TextIOWrapper)

        with pytest.raises(IOError):
            fd.read()

        fd.write(u"abc")
        assert fd.tell() == 3

    with smbclient.open_file(file_path, mode='a') as fd:
        assert fd.tell() == 3
        fd.write(u"def")
        assert fd.tell() == 6

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == u"abcdef" 

def next_filehandle(self):
        """Go to the next file and retrun its filehandle or None (meaning no more files)."""
        filename = self.next_filename()
        if filename is None:
            fhandle = None
        elif filename == '-':
            fhandle = io.TextIOWrapper(sys.stdin.buffer, encoding=self.encoding)
        elif filename == '<filehandle_input>':
            fhandle = self.filehandle
        else:
            filename_extension = filename.split('.')[-1]
            if filename_extension == 'gz':
                myopen = gzip.open
            elif filename_extension == 'xz':
                myopen = lzma.open
            elif filename_extension == 'bz2':
                myopen = bz2.open
            else:
                myopen = open
            fhandle = myopen(filename, 'rt', encoding=self.encoding)
        self.filehandle = fhandle
        return fhandle 

def popen(cmd, mode="r", buffering=-1):
    if not isinstance(cmd, str):
        raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
    if mode not in ("r", "w"):
        raise ValueError("invalid mode %r" % mode)
    if buffering == 0 or buffering is None:
        raise ValueError("popen() does not support unbuffered streams")
    import subprocess, io
    if mode == "r":
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdout=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
    else:
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdin=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdin), proc)

# Helper for popen() -- a proxy for a file whose close waits for the process 

def toprettyxml(self, indent="\t", newl="\n", encoding=None):
        if encoding is None:
            writer = io.StringIO()
        else:
            writer = io.TextIOWrapper(io.BytesIO(),
                                      encoding=encoding,
                                      errors="xmlcharrefreplace",
                                      newline='\n')
        if self.nodeType == Node.DOCUMENT_NODE:
            # Can pass encoding only to document, to put it into XML header
            self.writexml(writer, "", indent, newl, encoding)
        else:
            self.writexml(writer, "", indent, newl)
        if encoding is None:
            return writer.getvalue()
        else:
            return writer.detach().getvalue() 

def listImport(self, force=None, path=None):
    _list = request.url_rule.split('/')[2]
    file = request.files['file']
    force = request.form.get('force')
    count = wl.countWhitelist() if _list.lower == 'whitelist' else bl.countBlacklist()
    if (count == 0) | (not count) | (force == "f"):
      if _list.lower == 'whitelist':
        wl.dropWhitelist()
        wl.importWhitelist(TextIOWrapper(file.stream))
      else:
        bl.dropBlacklist()
        bl.importBlacklist(TextIOWrapper(file.stream))
      status = _list[0]+"l_imported"
    else:
      status = _list[0]+"l_already_filled"
    return render_template('admin.html', status=status, **self.adminInfo())


  # /admin/whitelist/export
  # /admin/blacklist/export 

def save_uploaded_file(request, process_records=None):

    if len(request.FILES) != 1:
        raise ValueError("Received %s files instead of 1" % len(request.FILES))

    # parse file
    stream = next(iter(request.FILES.values()))
    filename = stream._name

    if not filename.endswith('.xls') and not filename.endswith('.xlsx'):
        stream = TextIOWrapper(stream.file, encoding = 'utf-8')

    json_records = parse_file(filename, stream)
    if process_records:
        json_records = process_records(json_records, filename=filename)

    # save json to temporary file
    uploaded_file_id = hashlib.md5(str(json_records).encode('utf-8')).hexdigest()
    serialized_file_path = _compute_serialized_file_path(uploaded_file_id)
    with gzip.open(serialized_file_path, "wt") as f:
        json.dump(json_records, f)

    return uploaded_file_id, filename, json_records 

def request_list(
        self, identity: Identity
    ) -> Iterator[Sequence[Mapping[str, object]]]:
        team = self.team
        if not (isinstance(team, identity.team_type) and
                cast(str, identity.identifier).startswith(team.server_url)):
            return
        start = 0
        while True:
            response = self.request(
                identity,
                'GET',
                self.LIST_URL.format(self.team, start)
            )
            assert response.code == 200
            payload = json.load(io.TextIOWrapper(response, encoding='utf-8'))
            response.close()
            yield from payload['values']
            if payload['isLastPage']:
                break
            start = payload['nextPageStart'] 

def _python2_bz2open(fn, mode, encoding, newline):
    """Wrapper to open bz2 in text mode.

    Parameters
    ----------
    fn : str
        File name
    mode : {'r', 'w'}
        File mode. Note that bz2 Text files are not supported.
    encoding : str
        Ignored, text bz2 files not supported in Python2.
    newline : str
        Ignored, text bz2 files not supported in Python2.
    """
    import bz2

    _check_mode(mode, encoding, newline)

    if "t" in mode:
        # BZ2File is missing necessary functions for TextIOWrapper
        warnings.warn("Assuming latin1 encoding for bz2 text file in Python2",
                      RuntimeWarning, stacklevel=5)
        mode = mode.replace("t", "")
    return bz2.BZ2File(fn, mode) 

def process_species(self, limit):
        """
        Loop through the xml file and process the species.
        We add elements to the graph, and store the
        id-to-label in the label_hash dict.
        :param limit:
        :return:
        """
        myfile = '/'.join((self.rawdir, self.files['data']['file']))
        with gzip.open(myfile, 'rb') as readbin:
            filereader = io.TextIOWrapper(readbin, newline="")
            filereader.readline()  # remove the xml declaration line
            for event, elem in ET.iterparse(filereader):
                # Species ids are == NCBITaxon ids
                self.process_xml_table(
                    elem, 'Species_gb', self._process_species_table_row, limit) 

def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    """Custom parse function for gauge tpm from mysql.
        TPM: 87060.0
        TPM: 95220.0
        TPM: 93600.0
        TPM: 90000.0
    """
    new_metrics = []

    new_line = readline_with_check(input, EOF_line='end')

    if "TPM:" in new_line:
        regex = re.findall(r'TPM: (?P<tpm>\d*.\d*)', new_line)
        tpm = float(regex[0])

        new_metrics.append(Metric(metric_name_prefix + 'tpm', tpm,
                                  type=MetricType.GAUGE, labels=labels,
                                  help="TPM (transaction per minute) from mysql"))

    return new_metrics 

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', help='Config file')
    args = parser.parse_args()

    if args.config:
        with open(args.config) as input:
            config = json.load(input)
    else:
        config = {}

    input = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
    state = persist_lines(config, input)

    emit_state(state)
    logger.debug("Exiting normally") 

def __enter__(self):
        mode = self.mode.replace("t", "").replace("b", "") + "b"

        f = self.fs.open(self.path, mode=mode)

        self.fobjects = [f]

        if self.compression is not None:
            compress = compr[self.compression]
            f = compress(f, mode=mode[0])
            self.fobjects.append(f)

        if "b" not in self.mode:
            # assume, for example, that 'r' is equivalent to 'rt' as in builtin
            f = io.TextIOWrapper(
                f, encoding=self.encoding, errors=self.errors, newline=self.newline
            )
            self.fobjects.append(f)

        return self.fobjects[-1] 

def __init__(self, stream):  # type: (io.TextIOWrapper) -> None
        self._stream = stream 

def __init__(self, stream):  # type: (io.TextIOWrapper) -> None
        self._stream = stream

        if hasattr("stream", "seekable") and stream.seekable():
            stream.seek(0) 

def test_read_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    expected = "[NtStatus 0xc0000034] No such file or directory"
    with pytest.raises(SMBOSError, match=re.escape(expected)):
        smbclient.open_file(file_path, mode='rb')

    with smbclient.open_file(file_path, mode='wb') as fd:
        fd.write(file_contents.encode('utf-8'))

    with smbclient.open_file(file_path) as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False
        assert fd.encoding == locale.getpreferredencoding()
        assert fd.errors == 'strict'
        assert fd.line_buffering is False
        assert fd.name == file_path
        assert fd.newlines is None

        actual = fd.read()
        assert actual == file_contents

        actual = fd.read()
        assert actual == ""

        fd.seek(0)
        actual = fd.readlines()

        expected_lines = file_contents.split("\n")
        expected = [l + "\n" if idx != len(expected_lines) - 1 else l for idx, l in enumerate(expected_lines)]
        assert actual == expected

        assert int(fd.tell()) == len(file_contents)

        with pytest.raises(IOError):
            fd.write(u"Fail")

    assert fd.closed 

def test_write_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    with smbclient.open_file(file_path, mode='w') as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False

        with pytest.raises(IOError):
            fd.read()

        assert fd.tell() == 0
        fd.write(file_contents)
        assert int(fd.tell()) == (len(file_contents) - 1 + len(os.linesep))

    assert fd.closed is True

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == file_contents

    with smbclient.open_file(file_path, mode='w') as fd:
        assert fd.tell() == 0
        assert fd.write(u"abc")
        assert fd.tell() == 3

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == u"abc" 

def preprocess_generate(**kwargs):
    with ContentDir() as content:
        content.download('text8.zip', 'http://mattmahoney.net/dc/text8.zip')

    with ZipFile(content.filepath('text8.zip')) as zip_reader:
        with zip_reader.open('text8') as text8_file:
            text = io.TextIOWrapper(text8_file).read()
            dataset = build_dataset(text, **kwargs)
            train, valid, test = split_dataset(dataset, **kwargs)

            print('saving train data ...')
            save_tfrecord(content.filepath('generate.train.tfrecord'),
                          train,
                          verbose=True)

            print('saving valid data ...')
            save_tfrecord(content.filepath('generate.valid.tfrecord'),
                          valid,
                          verbose=True)

            print('saving test data ...')
            save_tfrecord(content.filepath('generate.test.tfrecord'),
                          test,
                          verbose=True)

            print('saving maps ...')
            np.savez(content.filepath('generate.map.npz'),
                     char_map=dataset['char_map'],
                     verbose=True)

            print('saving metadata ...')
            metadata = {
                'observations': {
                    'train': len(train['length']),
                    'valid': len(valid['length']),
                    'test': len(test['length'])
                }
            }
            with open(content.filepath('generate.meta.json'), 'w') as fp:
                json.dump(metadata, fp) 

def preprocess_autocomplete(**kwargs):
    with ContentDir() as content:
        content.download('text8.zip', 'http://mattmahoney.net/dc/text8.zip')

    with ZipFile(content.filepath('text8.zip')) as zip_reader:
        with zip_reader.open('text8') as text8_file:
            text = io.TextIOWrapper(text8_file).read()
            dataset = build_dataset(text, **kwargs)
            train, valid, test = split_dataset(dataset, **kwargs)

            print('saving train data ...')
            save_tfrecord(content.filepath('autocomplete.train.tfrecord'),
                          train,
                          verbose=True)

            print('saving valid data ...')
            save_tfrecord(content.filepath('autocomplete.valid.tfrecord'),
                          valid,
                          verbose=True)

            print('saving test data ...')
            save_tfrecord(content.filepath('autocomplete.test.tfrecord'),
                          test,
                          verbose=True)

            print('saving maps ...')
            np.savez(content.filepath('autocomplete.map.npz'),
                     word_map=dataset['word_map'],
                     char_map=dataset['char_map'],
                     verbose=True)

            print('saving metadata ...')
            metadata = {
                'observations': {
                    'train': len(train['length']),
                    'valid': len(valid['length']),
                    'test': len(test['length'])
                }
            }
            with open(content.filepath('autocomplete.meta.json'), 'w') as fp:
                json.dump(metadata, fp) 

def _bam_init(self, f, regions):
        pline = [self.samtools_path, 'view', '-H', f.name]
        try:
            p = Popen(pline, bufsize=-1, stdout=PIPE,
                      stderr=PIPE)
        except OSError:
            raise OSError('Samtools must be installed for BAM file support!\n')
        self.header_as_dict([line.decode('utf-8').rstrip('\n\r') for line in p.stdout])
        p.wait()
        if regions:
            try:
                open(''.join([f.name, '.bai']))
            except EnvironmentError:
                sys.stderr.write("BAM index not found. Attempting to index file.\n")
                index_p = Popen([self.samtools_path, 'index', f.name], stdout=PIPE, stderr=PIPE)

                _, err = index_p.communicate()
                if index_p.returncode > 0 or re.search("fail", str(err)):
                    raise OSError("Indexing failed. Is the BAM file sorted?\n")
                else:
                    sys.stderr.write("Index created successfully.\n")
            pline = [self.samtools_path, 'view', f.name, regions]
        else:
            pline = [self.samtools_path, 'view', f.name]
        self.p = Popen(pline, bufsize=-1, stdout=PIPE,
                  stderr=PIPE)
        if PY3:
            self.f = TextIOWrapper(self.p.stdout)
        else:
            self.f = self.p.stdout

        self._conn = 'proc' 

def _gettextwriter(out, encoding):
    if out is None:
        import sys
        return sys.stdout

    if isinstance(out, io.TextIOBase):
        # use a text writer as is
        return out

    # wrap a binary writer with TextIOWrapper
    if isinstance(out, io.RawIOBase):
        # Keep the original file open when the TextIOWrapper is
        # destroyed
        class _wrapper:
            __class__ = out.__class__
            def __getattr__(self, name):
                return getattr(out, name)
        buffer = _wrapper()
        buffer.close = lambda: None
    else:
        # This is to handle passed objects that aren't in the
        # IOBase hierarchy, but just have a write method
        buffer = io.BufferedIOBase()
        buffer.writable = lambda: True
        buffer.write = out.write
        try:
            # TextIOWrapper uses this methods to determine
            # if BOM (for UTF-16, etc) should be added
            buffer.seekable = out.seekable
            buffer.tell = out.tell
        except AttributeError:
            pass
    return io.TextIOWrapper(buffer, encoding=encoding,
                            errors='xmlcharrefreplace',
                            newline='\n',
                            write_through=True) 

def gzip_open_unicode(filename, mode="rb", compresslevel=9,
                      encoding='utf-8', fileobj=None, errors=None, newline=None):
    if fileobj is None:
        fileobj = GzipFile(filename, mode, compresslevel, fileobj)
    return io.TextIOWrapper(fileobj, encoding, errors, newline) 

def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def parse(self, fp, headersonly=False):
        """Create a message structure from the data in a binary file.

        Reads all the data from the file and returns the root of the message
        structure.  Optional headersonly is a flag specifying whether to stop
        parsing after reading the headers or not.  The default is False,
        meaning it parses the entire contents of the file.
        """
        fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape')
        with fp:
            return self.parser.parse(fp, headersonly) 

def popen(cmd, mode="rb"):
  if not isinstance(cmd, str):
    raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))

  import subprocess, io, threading

  # cleanup function for subprocesses,
  def cleanup(proc, cmd):
    ret = proc.wait()
    if ret > 0:
      raise SubprocessFailed('cmd %s returned %d !' % (cmd,ret))
    return

  # text-mode,
  if mode == "r":
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return io.TextIOWrapper(proc.stdout)
  elif mode == "w":
    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return io.TextIOWrapper(proc.stdin)
  # binary,
  elif mode == "rb":
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return proc.stdout
  elif mode == "wb":
    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return proc.stdin
  # sanity,
  else:
    raise ValueError("invalid mode %s" % mode) 

def set_binary_mode(fh):
    """ Helper method to set up binary mode for file handles.
    Emphasis being sys.stdin, sys.stdout, sys.stderr.
    For python3, we want to return .buffer
    For python2+windows we want to set os.O_BINARY
    """
    typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
    # check for file handle
    if not isinstance(fh, typefile):
        return fh

    # check for python3 and buffer
    if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
        return fh.buffer
    # check for python3
    elif sys.version_info >= (3, 0):
        pass
    # check for windows python2. SPL-175233 -- python3 stdout is already binary
    elif sys.platform == 'win32':
        # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
        # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
        # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
        from platform import python_implementation
        implementation = python_implementation()
        if implementation == 'PyPy':
            return os.fdopen(fh.fileno(), 'wb', 0)
        else:
            import msvcrt
            msvcrt.setmode(fh.fileno(), os.O_BINARY)
    return fh 

def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 

def set_binary_mode(fh):
    """ Helper method to set up binary mode for file handles.
    Emphasis being sys.stdin, sys.stdout, sys.stderr.
    For python3, we want to return .buffer
    For python2+windows we want to set os.O_BINARY
    """
    typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
    # check for file handle
    if not isinstance(fh, typefile):
        return fh

    # check for python3 and buffer
    if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
        return fh.buffer
    # check for python3
    elif sys.version_info >= (3, 0):
        pass
    # check for windows python2. SPL-175233 -- python3 stdout is already binary
    elif sys.platform == 'win32':
        # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
        # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
        # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
        from platform import python_implementation
        implementation = python_implementation()
        if implementation == 'PyPy':
            return os.fdopen(fh.fileno(), 'wb', 0)
        else:
            import msvcrt
            msvcrt.setmode(fh.fileno(), os.O_BINARY)
    return fh 

def parse(self, fp, headersonly=False):
        """Create a message structure from the data in a binary file.

        Reads all the data from the file and returns the root of the message
        structure.  Optional headersonly is a flag specifying whether to stop
        parsing after reading the headers or not.  The default is False,
        meaning it parses the entire contents of the file.
        """
        fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape')
        with fp:
            return self.parser.parse(fp, headersonly)