Python io.TextIOWrapper() Examples

def _fopen(fname, mode):
    """
    Extend file open function, to support 
        1) "-", which means stdin/stdout
        2) "$cmd |" which means pipe.stdout
    """
    if mode not in ["w", "r", "wb", "rb"]:
        raise ValueError("Unknown open mode: {mode}".format(mode=mode))
    if not fname:
        return None
    fname = fname.rstrip()
    if fname == "-":
        if mode in ["w", "wb"]:
            return sys.stdout.buffer if mode == "wb" else sys.stdout
        else:
            return sys.stdin.buffer if mode == "rb" else sys.stdin
    elif fname[-1] == "|":
        pin = pipe_fopen(fname[:-1], mode, background=(mode == "rb"))
        return pin if mode == "rb" else TextIOWrapper(pin)
    else:
        if mode in ["r", "rb"] and not os.path.exists(fname):
            raise FileNotFoundError(
                "Could not find common file: {}".format(fname))
        return open(fname, mode) 

def _python2_bz2open(fn, mode, encoding, newline):
    """Wrapper to open bz2 in text mode.

    Parameters
    ----------
    fn : str
        File name
    mode : {'r', 'w'}
        File mode. Note that bz2 Text files are not supported.
    encoding : str
        Ignored, text bz2 files not supported in Python2.
    newline : str
        Ignored, text bz2 files not supported in Python2.
    """
    import bz2

    _check_mode(mode, encoding, newline)

    if "t" in mode:
        # BZ2File is missing necessary functions for TextIOWrapper
        warnings.warn("Assuming latin1 encoding for bz2 text file in Python2",
                      RuntimeWarning, stacklevel=5)
        mode = mode.replace("t", "")
    return bz2.BZ2File(fn, mode) 

def __enter__(self):
        mode = self.mode.replace("t", "").replace("b", "") + "b"

        f = self.fs.open(self.path, mode=mode)

        self.fobjects = [f]

        if self.compression is not None:
            compress = compr[self.compression]
            f = compress(f, mode=mode[0])
            self.fobjects.append(f)

        if "b" not in self.mode:
            # assume, for example, that 'r' is equivalent to 'rt' as in builtin
            f = io.TextIOWrapper(
                f, encoding=self.encoding, errors=self.errors, newline=self.newline
            )
            self.fobjects.append(f)

        return self.fobjects[-1] 

def save_uploaded_file(request, process_records=None):

    if len(request.FILES) != 1:
        raise ValueError("Received %s files instead of 1" % len(request.FILES))

    # parse file
    stream = next(iter(request.FILES.values()))
    filename = stream._name

    if not filename.endswith('.xls') and not filename.endswith('.xlsx'):
        stream = TextIOWrapper(stream.file, encoding = 'utf-8')

    json_records = parse_file(filename, stream)
    if process_records:
        json_records = process_records(json_records, filename=filename)

    # save json to temporary file
    uploaded_file_id = hashlib.md5(str(json_records).encode('utf-8')).hexdigest()
    serialized_file_path = _compute_serialized_file_path(uploaded_file_id)
    with gzip.open(serialized_file_path, "wt") as f:
        json.dump(json_records, f)

    return uploaded_file_id, filename, json_records 

def test_write_exclusive_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    with smbclient.open_file(file_path, mode='x') as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False

        with pytest.raises(IOError):
            fd.read()

        assert fd.tell() == 0
        fd.write(file_contents)
        assert int(fd.tell()) == (len(file_contents) - 1 + len(os.linesep))

    assert fd.closed is True

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == file_contents

    with pytest.raises(OSError, match=re.escape("[NtStatus 0xc0000035] File exists: ")):
        smbclient.open_file(file_path, mode='x')

    assert fd.closed is True 

def test_append_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")

    with smbclient.open_file(file_path, mode='a') as fd:
        assert isinstance(fd, io.TextIOWrapper)

        with pytest.raises(IOError):
            fd.read()

        fd.write(u"abc")
        assert fd.tell() == 3

    with smbclient.open_file(file_path, mode='a') as fd:
        assert fd.tell() == 3
        fd.write(u"def")
        assert fd.tell() == 6

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == u"abcdef" 

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', help='Config file')
    args = parser.parse_args()

    if args.config:
        with open(args.config) as input:
            config = json.load(input)
    else:
        config = {}

    input = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
    state = persist_lines(config, input)

    emit_state(state)
    logger.debug("Exiting normally") 

def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    """Custom parse function for gauge tpm from mysql.
        TPM: 87060.0
        TPM: 95220.0
        TPM: 93600.0
        TPM: 90000.0
    """
    new_metrics = []

    new_line = readline_with_check(input, EOF_line='end')

    if "TPM:" in new_line:
        regex = re.findall(r'TPM: (?P<tpm>\d*.\d*)', new_line)
        tpm = float(regex[0])

        new_metrics.append(Metric(metric_name_prefix + 'tpm', tpm,
                                  type=MetricType.GAUGE, labels=labels,
                                  help="TPM (transaction per minute) from mysql"))

    return new_metrics 

def process_species(self, limit):
        """
        Loop through the xml file and process the species.
        We add elements to the graph, and store the
        id-to-label in the label_hash dict.
        :param limit:
        :return:
        """
        myfile = '/'.join((self.rawdir, self.files['data']['file']))
        with gzip.open(myfile, 'rb') as readbin:
            filereader = io.TextIOWrapper(readbin, newline="")
            filereader.readline()  # remove the xml declaration line
            for event, elem in ET.iterparse(filereader):
                # Species ids are == NCBITaxon ids
                self.process_xml_table(
                    elem, 'Species_gb', self._process_species_table_row, limit) 

def next_filehandle(self):
        """Go to the next file and retrun its filehandle or None (meaning no more files)."""
        filename = self.next_filename()
        if filename is None:
            fhandle = None
        elif filename == '-':
            fhandle = io.TextIOWrapper(sys.stdin.buffer, encoding=self.encoding)
        elif filename == '<filehandle_input>':
            fhandle = self.filehandle
        else:
            filename_extension = filename.split('.')[-1]
            if filename_extension == 'gz':
                myopen = gzip.open
            elif filename_extension == 'xz':
                myopen = lzma.open
            elif filename_extension == 'bz2':
                myopen = bz2.open
            else:
                myopen = open
            fhandle = myopen(filename, 'rt', encoding=self.encoding)
        self.filehandle = fhandle
        return fhandle 

def popen(cmd, mode="r", buffering=-1):
    if not isinstance(cmd, str):
        raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
    if mode not in ("r", "w"):
        raise ValueError("invalid mode %r" % mode)
    if buffering == 0 or buffering is None:
        raise ValueError("popen() does not support unbuffered streams")
    import subprocess, io
    if mode == "r":
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdout=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
    else:
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdin=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdin), proc)

# Helper for popen() -- a proxy for a file whose close waits for the process 

def request_list(
        self, identity: Identity
    ) -> Iterator[Sequence[Mapping[str, object]]]:
        team = self.team
        if not (isinstance(team, identity.team_type) and
                cast(str, identity.identifier).startswith(team.server_url)):
            return
        start = 0
        while True:
            response = self.request(
                identity,
                'GET',
                self.LIST_URL.format(self.team, start)
            )
            assert response.code == 200
            payload = json.load(io.TextIOWrapper(response, encoding='utf-8'))
            response.close()
            yield from payload['values']
            if payload['isLastPage']:
                break
            start = payload['nextPageStart'] 

def toprettyxml(self, indent="\t", newl="\n", encoding=None):
        if encoding is None:
            writer = io.StringIO()
        else:
            writer = io.TextIOWrapper(io.BytesIO(),
                                      encoding=encoding,
                                      errors="xmlcharrefreplace",
                                      newline='\n')
        if self.nodeType == Node.DOCUMENT_NODE:
            # Can pass encoding only to document, to put it into XML header
            self.writexml(writer, "", indent, newl, encoding)
        else:
            self.writexml(writer, "", indent, newl)
        if encoding is None:
            return writer.getvalue()
        else:
            return writer.detach().getvalue() 

def listImport(self, force=None, path=None):
    _list = request.url_rule.split('/')[2]
    file = request.files['file']
    force = request.form.get('force')
    count = wl.countWhitelist() if _list.lower == 'whitelist' else bl.countBlacklist()
    if (count == 0) | (not count) | (force == "f"):
      if _list.lower == 'whitelist':
        wl.dropWhitelist()
        wl.importWhitelist(TextIOWrapper(file.stream))
      else:
        bl.dropBlacklist()
        bl.importBlacklist(TextIOWrapper(file.stream))
      status = _list[0]+"l_imported"
    else:
      status = _list[0]+"l_already_filled"
    return render_template('admin.html', status=status, **self.adminInfo())


  # /admin/whitelist/export
  # /admin/blacklist/export 

def write(self, x):
            if isinstance(x, str) or is_bytes(x):
                try:
                    self.flush()
                except Exception:
                    pass
                return self.buffer.write(str(x))
            return io.TextIOWrapper.write(self, x) 

def __init__(self, stream, encoding, errors,
                 force_readable=False, force_writable=False, **extra):
        self._stream = stream = _FixupStream(stream, force_readable,
                                             force_writable)
        io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra)

    # The io module is a place where the Python 3 text behavior
    # was forced upon Python 2, so we need to unbreak
    # it to look like Python 2. 

def write(self, x):
            if isinstance(x, str) or is_bytes(x):
                try:
                    self.flush()
                except Exception:
                    pass
                return self.buffer.write(str(x))
            return io.TextIOWrapper.write(self, x) 

def process_gene_ids(self, limit):
        raw = '/'.join((self.rawdir, self.files['gene_ids']['file']))

        graph = self.graph

        model = Model(graph)
        LOG.info("Processing: %s", self.files['gene_ids']['file'])
        line_counter = 0
        geno = Genotype(graph)
        with gzip.open(raw, 'rb') as csvfile:
            filereader = csv.reader(
                io.TextIOWrapper(csvfile, newline=""), delimiter=',',
                quotechar='\"')
            for row in filereader:
                line_counter += 1
                (taxon_num,
                 gene_num,
                 gene_symbol,
                 gene_synonym,
                 live,
                 gene_type) = row
                # 6239,WBGene00000001,aap-1,Y110A7A.10,Live,protein_coding_gene

                taxon_id = 'NCBITaxon:'+taxon_num
                gene_id = 'WormBase:'+gene_num
                if gene_symbol == '':
                    gene_symbol = gene_synonym
                if gene_symbol == '':
                    gene_symbol = None
                model.addClassToGraph(
                    gene_id, gene_symbol, self.globaltt['gene'])
                if live == 'Dead':
                    model.addDeprecatedClass(gene_id)
                geno.addTaxon(taxon_id, gene_id)
                if gene_synonym != '' and gene_synonym is not None:
                    model.addSynonym(gene_id, gene_synonym)

                if limit is not None and line_counter > limit:
                    break 

def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    """
    Custom parse function for specjbb.
    For sample output from specjbb see file:
    ./specjbb_sample_stdout.txt

    Discards until finds
    >>Response times:<<
    and read until empty line.

    Readed lines represents a table.
    In the code the table representation is named data_frame.
    """
    new_metrics = []
    input_lines = []

    # discarding lines
    new_line = readline_with_check(input)
    while not re.match(r"^\s*Response times:\s*$", new_line):
        new_line = readline_with_check(input)
    new_line = readline_with_check(input)

    # reading until empty line
    while not re.match(EMPTY_LINE, new_line):
        input_lines.append(new_line)
        new_line = readline_with_check(input)
    log.debug("Found separator in {0}".format(new_line))

    # Two dimensional list, first row contains names of columns. Almost as data frame.
    data_frame = [[el.strip() for el in line.split(",")] for line in input_lines]

    # For now we need only one metric: TotalPurchase, p99.
    metric_name = metric_name_prefix + 'p99_total_purchase'
    metric_value = float(data_frame[1][-3])  # total purchase, p99
    new_metrics.append(Metric(metric_name, metric_value,
                              type=MetricType.GAUGE, labels=labels,
                              help="Specjbb2015 metric, Total Purchase, percentile 99"))

    return new_metrics 

def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    return [Metric(name="example", value=1.337, labels={"test": "label"}, type=MetricType.GAUGE,
                   help="Empty example metric")] 

def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    """Custom parse function for cassandra-stress.
        Results:
        Op rate                   :   14,997 op/s  [WRITE: 14,997 op/s]
        Partition rate            :   14,997 pk/s  [WRITE: 14,997 pk/s]
        Row rate                  :   14,997 row/s [WRITE: 14,997 row/s]
        Latency mean              :    1.9 ms [WRITE: 1.9 ms]
        Latency median            :    0.3 ms [WRITE: 0.3 ms]
        Latency 95th percentile   :    0.4 ms [WRITE: 0.4 ms]
        Latency 99th percentile   :   74.0 ms [WRITE: 74.0 ms]
        Latency 99.9th percentile :  146.8 ms [WRITE: 146.8 ms]
        Latency max               :  160.2 ms [WRITE: 160.2 ms]
        Total partitions          :  1,350,028 [WRITE: 1,350,028]
        Total errors              :          0 [WRITE: 0]
        Total GC count            : 0
        Total GC memory           : 0.000 KiB
        Total GC time             :    0.0 seconds
        Avg GC time               :    NaN ms
        StdDev GC time            :    0.0 ms
        Total operation time      : 00:01:30
    """

    new_metrics = []
    new_line = readline_with_check(input, EOF_line)
    if "Op rate" in new_line:
        read_op_rate = re.search(r'Op rate[ ]*:[ ]*([0-9,]*) op/s', new_line)
        op_rate = float(''.join(read_op_rate.group(1).split(',')))
        new_metrics.append(Metric(metric_name_prefix + 'qps', op_rate,
                                  type=MetricType.GAUGE, labels=labels,
                                  help="QPS"))
    if "Latency 99th percentile" in new_line:
        read = re.search(
            r'Latency 99th percentile[ ]*:[ ]*([0-9]*\.[0-9]*) ms', new_line)
        p99 = float(read.group(1))
        new_metrics.append(Metric(metric_name_prefix + 'p99', p99,
                                  type=MetricType.GAUGE, labels=labels,
                                  help="99th percentile"))
    return new_metrics 

def _wrap_reader_for_text(fp, encoding):
    if isinstance(fp.read(0), bytes):
        fp = io.TextIOWrapper(io.BufferedReader(fp), encoding)
    return fp 

def test_sniff_delimiter_encoding(python_parser_only, encoding):
    parser = python_parser_only
    data = """ignore this
ignore this too
index|A|B|C
foo|1|2|3
bar|4|5|6
baz|7|8|9
"""

    if encoding is not None:
        data = u(data).encode(encoding)
        data = BytesIO(data)

        if compat.PY3:
            from io import TextIOWrapper
            data = TextIOWrapper(data, encoding=encoding)
    else:
        data = StringIO(data)

    result = parser.read_csv(data, index_col=0, sep=None,
                             skiprows=2, encoding=encoding)
    expected = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]],
                         columns=["A", "B", "C"],
                         index=Index(["foo", "bar", "baz"], name="index"))
    tm.assert_frame_equal(result, expected) 

def test_buffer_rd_bytes_bad_unicode(c_parser_only):
    # see gh-22748
    parser = c_parser_only
    t = BytesIO(b"\xB0")

    if PY3:
        msg = "'utf-8' codec can't encode character"
        t = TextIOWrapper(t, encoding="ascii", errors="surrogateescape")
    else:
        msg = "'utf8' codec can't decode byte"

    with pytest.raises(UnicodeError, match=msg):
        parser.read_csv(t, encoding="UTF-8") 

def _wrap_writer_for_text(fp, encoding):
    try:
        fp.write('')
    except TypeError:
        fp = io.TextIOWrapper(fp, encoding)
    return fp 

def _wrap_reader_for_text(fp, encoding):
    if isinstance(fp.read(0), bytes):
        fp = io.TextIOWrapper(io.BufferedReader(fp), encoding)
    return fp 

def load(self) -> PKey:
        try:
            obj = self.driver.get_object(self.container.name, self.object_name)
        except ObjectDoesNotExistError:
            raise EmptyStoreError()
        with io.BytesIO() as buffer_:
            for chunk in self.driver.download_object_as_stream(obj):
                if isinstance(chunk, str):  # DummyDriver yields str, not bytes
                    chunk = chunk.encode()
                buffer_.write(chunk)
            buffer_.seek(0)
            with io.TextIOWrapper(buffer_) as tio:
                return read_private_key_file(tio) 

def open(filename):
    """Open a file in read only mode using the encoding detected by
    detect_encoding().
    """
    buffer = _builtin_open(filename, 'rb')
    try:
        encoding, lines = detect_encoding(buffer.readline)
        buffer.seek(0)
        text = TextIOWrapper(buffer, encoding, line_buffering=True)
        text.mode = 'r'
        return text
    except:
        buffer.close()
        raise 

def open(filename):
    """Open a file in read only mode using the encoding detected by
    detect_encoding().
    """
    buffer = _builtin_open(filename, 'rb')
    try:
        encoding, lines = detect_encoding(buffer.readline)
        buffer.seek(0)
        text = TextIOWrapper(buffer, encoding, line_buffering=True)
        text.mode = 'r'
        return text
    except:
        buffer.close()
        raise 

def stdin_get_value():
        """Read the value from stdin."""
        return TextIOWrapper(sys.stdin.buffer, errors='ignore').read()