Python io.TextIOWrapper() Examples

The following are 30 code examples for showing how to use io.TextIOWrapper(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module io , or try the search function .

Example 1
Project: kaldi-python-io   Author: funcwj   File: inst.py    License: Apache License 2.0 6 votes vote down vote up
def _fopen(fname, mode):
    """
    Extend file open function, to support 
        1) "-", which means stdin/stdout
        2) "$cmd |" which means pipe.stdout
    """
    if mode not in ["w", "r", "wb", "rb"]:
        raise ValueError("Unknown open mode: {mode}".format(mode=mode))
    if not fname:
        return None
    fname = fname.rstrip()
    if fname == "-":
        if mode in ["w", "wb"]:
            return sys.stdout.buffer if mode == "wb" else sys.stdout
        else:
            return sys.stdin.buffer if mode == "rb" else sys.stdin
    elif fname[-1] == "|":
        pin = pipe_fopen(fname[:-1], mode, background=(mode == "rb"))
        return pin if mode == "rb" else TextIOWrapper(pin)
    else:
        if mode in ["r", "rb"] and not os.path.exists(fname):
            raise FileNotFoundError(
                "Could not find common file: {}".format(fname))
        return open(fname, mode) 
Example 2
Project: smbprotocol   Author: jborean93   File: test_smbclient_os.py    License: MIT License 6 votes vote down vote up
def test_write_exclusive_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    with smbclient.open_file(file_path, mode='x') as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False

        with pytest.raises(IOError):
            fd.read()

        assert fd.tell() == 0
        fd.write(file_contents)
        assert int(fd.tell()) == (len(file_contents) - 1 + len(os.linesep))

    assert fd.closed is True

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == file_contents

    with pytest.raises(OSError, match=re.escape("[NtStatus 0xc0000035] File exists: ")):
        smbclient.open_file(file_path, mode='x')

    assert fd.closed is True 
Example 3
Project: smbprotocol   Author: jborean93   File: test_smbclient_os.py    License: MIT License 6 votes vote down vote up
def test_append_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")

    with smbclient.open_file(file_path, mode='a') as fd:
        assert isinstance(fd, io.TextIOWrapper)

        with pytest.raises(IOError):
            fd.read()

        fd.write(u"abc")
        assert fd.tell() == 3

    with smbclient.open_file(file_path, mode='a') as fd:
        assert fd.tell() == 3
        fd.write(u"def")
        assert fd.tell() == 6

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == u"abcdef" 
Example 4
Project: udapi-python   Author: udapi   File: files.py    License: GNU General Public License v3.0 6 votes vote down vote up
def next_filehandle(self):
        """Go to the next file and retrun its filehandle or None (meaning no more files)."""
        filename = self.next_filename()
        if filename is None:
            fhandle = None
        elif filename == '-':
            fhandle = io.TextIOWrapper(sys.stdin.buffer, encoding=self.encoding)
        elif filename == '<filehandle_input>':
            fhandle = self.filehandle
        else:
            filename_extension = filename.split('.')[-1]
            if filename_extension == 'gz':
                myopen = gzip.open
            elif filename_extension == 'xz':
                myopen = lzma.open
            elif filename_extension == 'bz2':
                myopen = bz2.open
            else:
                myopen = open
            fhandle = myopen(filename, 'rt', encoding=self.encoding)
        self.filehandle = fhandle
        return fhandle 
Example 5
Project: jawfish   Author: war-and-code   File: os.py    License: MIT License 6 votes vote down vote up
def popen(cmd, mode="r", buffering=-1):
    if not isinstance(cmd, str):
        raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
    if mode not in ("r", "w"):
        raise ValueError("invalid mode %r" % mode)
    if buffering == 0 or buffering is None:
        raise ValueError("popen() does not support unbuffered streams")
    import subprocess, io
    if mode == "r":
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdout=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
    else:
        proc = subprocess.Popen(cmd,
                                shell=True,
                                stdin=subprocess.PIPE,
                                bufsize=buffering)
        return _wrap_close(io.TextIOWrapper(proc.stdin), proc)

# Helper for popen() -- a proxy for a file whose close waits for the process 
Example 6
Project: jawfish   Author: war-and-code   File: minidom.py    License: MIT License 6 votes vote down vote up
def toprettyxml(self, indent="\t", newl="\n", encoding=None):
        if encoding is None:
            writer = io.StringIO()
        else:
            writer = io.TextIOWrapper(io.BytesIO(),
                                      encoding=encoding,
                                      errors="xmlcharrefreplace",
                                      newline='\n')
        if self.nodeType == Node.DOCUMENT_NODE:
            # Can pass encoding only to document, to put it into XML header
            self.writexml(writer, "", indent, newl, encoding)
        else:
            self.writexml(writer, "", indent, newl)
        if encoding is None:
            return writer.getvalue()
        else:
            return writer.detach().getvalue() 
Example 7
Project: watchdog   Author: flipkart-incubator   File: index.py    License: Apache License 2.0 6 votes vote down vote up
def listImport(self, force=None, path=None):
    _list = request.url_rule.split('/')[2]
    file = request.files['file']
    force = request.form.get('force')
    count = wl.countWhitelist() if _list.lower == 'whitelist' else bl.countBlacklist()
    if (count == 0) | (not count) | (force == "f"):
      if _list.lower == 'whitelist':
        wl.dropWhitelist()
        wl.importWhitelist(TextIOWrapper(file.stream))
      else:
        bl.dropBlacklist()
        bl.importBlacklist(TextIOWrapper(file.stream))
      status = _list[0]+"l_imported"
    else:
      status = _list[0]+"l_already_filled"
    return render_template('admin.html', status=status, **self.adminInfo())


  # /admin/whitelist/export
  # /admin/blacklist/export 
Example 8
Project: seqr   Author: macarthur-lab   File: file_utils.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def save_uploaded_file(request, process_records=None):

    if len(request.FILES) != 1:
        raise ValueError("Received %s files instead of 1" % len(request.FILES))

    # parse file
    stream = next(iter(request.FILES.values()))
    filename = stream._name

    if not filename.endswith('.xls') and not filename.endswith('.xlsx'):
        stream = TextIOWrapper(stream.file, encoding = 'utf-8')

    json_records = parse_file(filename, stream)
    if process_records:
        json_records = process_records(json_records, filename=filename)

    # save json to temporary file
    uploaded_file_id = hashlib.md5(str(json_records).encode('utf-8')).hexdigest()
    serialized_file_path = _compute_serialized_file_path(uploaded_file_id)
    with gzip.open(serialized_file_path, "wt") as f:
        json.dump(json_records, f)

    return uploaded_file_id, filename, json_records 
Example 9
Project: geofront   Author: spoqa   File: stash.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def request_list(
        self, identity: Identity
    ) -> Iterator[Sequence[Mapping[str, object]]]:
        team = self.team
        if not (isinstance(team, identity.team_type) and
                cast(str, identity.identifier).startswith(team.server_url)):
            return
        start = 0
        while True:
            response = self.request(
                identity,
                'GET',
                self.LIST_URL.format(self.team, start)
            )
            assert response.code == 200
            payload = json.load(io.TextIOWrapper(response, encoding='utf-8'))
            response.close()
            yield from payload['values']
            if payload['isLastPage']:
                break
            start = payload['nextPageStart'] 
Example 10
Project: recruit   Author: Frank-qlu   File: _datasource.py    License: Apache License 2.0 6 votes vote down vote up
def _python2_bz2open(fn, mode, encoding, newline):
    """Wrapper to open bz2 in text mode.

    Parameters
    ----------
    fn : str
        File name
    mode : {'r', 'w'}
        File mode. Note that bz2 Text files are not supported.
    encoding : str
        Ignored, text bz2 files not supported in Python2.
    newline : str
        Ignored, text bz2 files not supported in Python2.
    """
    import bz2

    _check_mode(mode, encoding, newline)

    if "t" in mode:
        # BZ2File is missing necessary functions for TextIOWrapper
        warnings.warn("Assuming latin1 encoding for bz2 text file in Python2",
                      RuntimeWarning, stacklevel=5)
        mode = mode.replace("t", "")
    return bz2.BZ2File(fn, mode) 
Example 11
Project: dipper   Author: monarch-initiative   File: OMIA.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def process_species(self, limit):
        """
        Loop through the xml file and process the species.
        We add elements to the graph, and store the
        id-to-label in the label_hash dict.
        :param limit:
        :return:
        """
        myfile = '/'.join((self.rawdir, self.files['data']['file']))
        with gzip.open(myfile, 'rb') as readbin:
            filereader = io.TextIOWrapper(readbin, newline="")
            filereader.readline()  # remove the xml declaration line
            for event, elem in ET.iterparse(filereader):
                # Species ids are == NCBITaxon ids
                self.process_xml_table(
                    elem, 'Species_gb', self._process_species_table_row, limit) 
Example 12
Project: workload-collocation-agent   Author: intel   File: parser_mysql_tpm_gauge.py    License: Apache License 2.0 6 votes vote down vote up
def parse(input: TextIOWrapper, regexp: str, separator: str = None,
          labels: Dict[str, str] = {}, metric_name_prefix: str = '') -> List[Metric]:
    """Custom parse function for gauge tpm from mysql.
        TPM: 87060.0
        TPM: 95220.0
        TPM: 93600.0
        TPM: 90000.0
    """
    new_metrics = []

    new_line = readline_with_check(input, EOF_line='end')

    if "TPM:" in new_line:
        regex = re.findall(r'TPM: (?P<tpm>\d*.\d*)', new_line)
        tpm = float(regex[0])

        new_metrics.append(Metric(metric_name_prefix + 'tpm', tpm,
                                  type=MetricType.GAUGE, labels=labels,
                                  help="TPM (transaction per minute) from mysql"))

    return new_metrics 
Example 13
Project: target-postgres   Author: statsbotco   File: __init__.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', help='Config file')
    args = parser.parse_args()

    if args.config:
        with open(args.config) as input:
            config = json.load(input)
    else:
        config = {}

    input = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
    state = persist_lines(config, input)

    emit_state(state)
    logger.debug("Exiting normally") 
Example 14
Project: filesystem_spec   Author: intake   File: core.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def __enter__(self):
        mode = self.mode.replace("t", "").replace("b", "") + "b"

        f = self.fs.open(self.path, mode=mode)

        self.fobjects = [f]

        if self.compression is not None:
            compress = compr[self.compression]
            f = compress(f, mode=mode[0])
            self.fobjects.append(f)

        if "b" not in self.mode:
            # assume, for example, that 'r' is equivalent to 'rt' as in builtin
            f = io.TextIOWrapper(
                f, encoding=self.encoding, errors=self.errors, newline=self.newline
            )
            self.fobjects.append(f)

        return self.fobjects[-1] 
Example 15
Project: clikit   Author: sdispater   File: stream_output_stream.py    License: MIT License 5 votes vote down vote up
def __init__(self, stream):  # type: (io.TextIOWrapper) -> None
        self._stream = stream 
Example 16
Project: clikit   Author: sdispater   File: stream_input_stream.py    License: MIT License 5 votes vote down vote up
def __init__(self, stream):  # type: (io.TextIOWrapper) -> None
        self._stream = stream

        if hasattr("stream", "seekable") and stream.seekable():
            stream.seek(0) 
Example 17
Project: smbprotocol   Author: jborean93   File: test_smbclient_os.py    License: MIT License 5 votes vote down vote up
def test_read_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    expected = "[NtStatus 0xc0000034] No such file or directory"
    with pytest.raises(SMBOSError, match=re.escape(expected)):
        smbclient.open_file(file_path, mode='rb')

    with smbclient.open_file(file_path, mode='wb') as fd:
        fd.write(file_contents.encode('utf-8'))

    with smbclient.open_file(file_path) as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False
        assert fd.encoding == locale.getpreferredencoding()
        assert fd.errors == 'strict'
        assert fd.line_buffering is False
        assert fd.name == file_path
        assert fd.newlines is None

        actual = fd.read()
        assert actual == file_contents

        actual = fd.read()
        assert actual == ""

        fd.seek(0)
        actual = fd.readlines()

        expected_lines = file_contents.split("\n")
        expected = [l + "\n" if idx != len(expected_lines) - 1 else l for idx, l in enumerate(expected_lines)]
        assert actual == expected

        assert int(fd.tell()) == len(file_contents)

        with pytest.raises(IOError):
            fd.write(u"Fail")

    assert fd.closed 
Example 18
Project: smbprotocol   Author: jborean93   File: test_smbclient_os.py    License: MIT License 5 votes vote down vote up
def test_write_text_file(smb_share):
    file_path = "%s\\%s" % (smb_share, "file.txt")
    file_contents = u"File Contents\nNewline"

    with smbclient.open_file(file_path, mode='w') as fd:
        assert isinstance(fd, io.TextIOWrapper)
        assert fd.closed is False

        with pytest.raises(IOError):
            fd.read()

        assert fd.tell() == 0
        fd.write(file_contents)
        assert int(fd.tell()) == (len(file_contents) - 1 + len(os.linesep))

    assert fd.closed is True

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == file_contents

    with smbclient.open_file(file_path, mode='w') as fd:
        assert fd.tell() == 0
        assert fd.write(u"abc")
        assert fd.tell() == 3

    with smbclient.open_file(file_path, mode='r') as fd:
        assert fd.read() == u"abc" 
Example 19
Project: post--memorization-in-rnns   Author: distillpub   File: generate.py    License: MIT License 5 votes vote down vote up
def preprocess_generate(**kwargs):
    with ContentDir() as content:
        content.download('text8.zip', 'http://mattmahoney.net/dc/text8.zip')

    with ZipFile(content.filepath('text8.zip')) as zip_reader:
        with zip_reader.open('text8') as text8_file:
            text = io.TextIOWrapper(text8_file).read()
            dataset = build_dataset(text, **kwargs)
            train, valid, test = split_dataset(dataset, **kwargs)

            print('saving train data ...')
            save_tfrecord(content.filepath('generate.train.tfrecord'),
                          train,
                          verbose=True)

            print('saving valid data ...')
            save_tfrecord(content.filepath('generate.valid.tfrecord'),
                          valid,
                          verbose=True)

            print('saving test data ...')
            save_tfrecord(content.filepath('generate.test.tfrecord'),
                          test,
                          verbose=True)

            print('saving maps ...')
            np.savez(content.filepath('generate.map.npz'),
                     char_map=dataset['char_map'],
                     verbose=True)

            print('saving metadata ...')
            metadata = {
                'observations': {
                    'train': len(train['length']),
                    'valid': len(valid['length']),
                    'test': len(test['length'])
                }
            }
            with open(content.filepath('generate.meta.json'), 'w') as fp:
                json.dump(metadata, fp) 
Example 20
Project: post--memorization-in-rnns   Author: distillpub   File: autocomplete.py    License: MIT License 5 votes vote down vote up
def preprocess_autocomplete(**kwargs):
    with ContentDir() as content:
        content.download('text8.zip', 'http://mattmahoney.net/dc/text8.zip')

    with ZipFile(content.filepath('text8.zip')) as zip_reader:
        with zip_reader.open('text8') as text8_file:
            text = io.TextIOWrapper(text8_file).read()
            dataset = build_dataset(text, **kwargs)
            train, valid, test = split_dataset(dataset, **kwargs)

            print('saving train data ...')
            save_tfrecord(content.filepath('autocomplete.train.tfrecord'),
                          train,
                          verbose=True)

            print('saving valid data ...')
            save_tfrecord(content.filepath('autocomplete.valid.tfrecord'),
                          valid,
                          verbose=True)

            print('saving test data ...')
            save_tfrecord(content.filepath('autocomplete.test.tfrecord'),
                          test,
                          verbose=True)

            print('saving maps ...')
            np.savez(content.filepath('autocomplete.map.npz'),
                     word_map=dataset['word_map'],
                     char_map=dataset['char_map'],
                     verbose=True)

            print('saving metadata ...')
            metadata = {
                'observations': {
                    'train': len(train['length']),
                    'valid': len(valid['length']),
                    'test': len(test['length'])
                }
            }
            with open(content.filepath('autocomplete.meta.json'), 'w') as fp:
                json.dump(metadata, fp) 
Example 21
Project: simplesam   Author: mdshw5   File: simplesam.py    License: MIT License 5 votes vote down vote up
def _bam_init(self, f, regions):
        pline = [self.samtools_path, 'view', '-H', f.name]
        try:
            p = Popen(pline, bufsize=-1, stdout=PIPE,
                      stderr=PIPE)
        except OSError:
            raise OSError('Samtools must be installed for BAM file support!\n')
        self.header_as_dict([line.decode('utf-8').rstrip('\n\r') for line in p.stdout])
        p.wait()
        if regions:
            try:
                open(''.join([f.name, '.bai']))
            except EnvironmentError:
                sys.stderr.write("BAM index not found. Attempting to index file.\n")
                index_p = Popen([self.samtools_path, 'index', f.name], stdout=PIPE, stderr=PIPE)

                _, err = index_p.communicate()
                if index_p.returncode > 0 or re.search("fail", str(err)):
                    raise OSError("Indexing failed. Is the BAM file sorted?\n")
                else:
                    sys.stderr.write("Index created successfully.\n")
            pline = [self.samtools_path, 'view', f.name, regions]
        else:
            pline = [self.samtools_path, 'view', f.name]
        self.p = Popen(pline, bufsize=-1, stdout=PIPE,
                  stderr=PIPE)
        if PY3:
            self.f = TextIOWrapper(self.p.stdout)
        else:
            self.f = self.p.stdout

        self._conn = 'proc' 
Example 22
Project: jawfish   Author: war-and-code   File: saxutils.py    License: MIT License 5 votes vote down vote up
def _gettextwriter(out, encoding):
    if out is None:
        import sys
        return sys.stdout

    if isinstance(out, io.TextIOBase):
        # use a text writer as is
        return out

    # wrap a binary writer with TextIOWrapper
    if isinstance(out, io.RawIOBase):
        # Keep the original file open when the TextIOWrapper is
        # destroyed
        class _wrapper:
            __class__ = out.__class__
            def __getattr__(self, name):
                return getattr(out, name)
        buffer = _wrapper()
        buffer.close = lambda: None
    else:
        # This is to handle passed objects that aren't in the
        # IOBase hierarchy, but just have a write method
        buffer = io.BufferedIOBase()
        buffer.writable = lambda: True
        buffer.write = out.write
        try:
            # TextIOWrapper uses this methods to determine
            # if BOM (for UTF-16, etc) should be added
            buffer.seekable = out.seekable
            buffer.tell = out.tell
        except AttributeError:
            pass
    return io.TextIOWrapper(buffer, encoding=encoding,
                            errors='xmlcharrefreplace',
                            newline='\n',
                            write_through=True) 
Example 23
Project: razzy-spinner   Author: rafasashi   File: data.py    License: GNU General Public License v3.0 5 votes vote down vote up
def gzip_open_unicode(filename, mode="rb", compresslevel=9,
                      encoding='utf-8', fileobj=None, errors=None, newline=None):
    if fileobj is None:
        fileobj = GzipFile(filename, mode, compresslevel, fileobj)
    return io.TextIOWrapper(fileobj, encoding, errors, newline) 
Example 24
Project: gist-alfred   Author: danielecook   File: makefile.py    License: MIT License 5 votes vote down vote up
def backport_makefile(self, mode="r", buffering=None, encoding=None,
                      errors=None, newline=None):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError(
            "invalid mode %r (only r, w, b allowed)" % (mode,)
        )
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example 25
Project: verge3d-blender-addon   Author: Soft8Soft   File: parser.py    License: GNU General Public License v3.0 5 votes vote down vote up
def parse(self, fp, headersonly=False):
        """Create a message structure from the data in a binary file.

        Reads all the data from the file and returns the root of the message
        structure.  Optional headersonly is a flag specifying whether to stop
        parsing after reading the headers or not.  The default is False,
        meaning it parses the entire contents of the file.
        """
        fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape')
        with fp:
            return self.parser.parse(fp, headersonly) 
Example 26
Project: Attentive-Filtering-Network   Author: jefflai108   File: kaldi_io.py    License: MIT License 5 votes vote down vote up
def popen(cmd, mode="rb"):
  if not isinstance(cmd, str):
    raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))

  import subprocess, io, threading

  # cleanup function for subprocesses,
  def cleanup(proc, cmd):
    ret = proc.wait()
    if ret > 0:
      raise SubprocessFailed('cmd %s returned %d !' % (cmd,ret))
    return

  # text-mode,
  if mode == "r":
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return io.TextIOWrapper(proc.stdout)
  elif mode == "w":
    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return io.TextIOWrapper(proc.stdin)
  # binary,
  elif mode == "rb":
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return proc.stdout
  elif mode == "wb":
    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
    threading.Thread(target=cleanup,args=(proc,cmd)).start() # clean-up thread,
    return proc.stdin
  # sanity,
  else:
    raise ValueError("invalid mode %s" % mode) 
Example 27
Project: misp42splunk   Author: remg427   File: internals.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def set_binary_mode(fh):
    """ Helper method to set up binary mode for file handles.
    Emphasis being sys.stdin, sys.stdout, sys.stderr.
    For python3, we want to return .buffer
    For python2+windows we want to set os.O_BINARY
    """
    typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
    # check for file handle
    if not isinstance(fh, typefile):
        return fh

    # check for python3 and buffer
    if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
        return fh.buffer
    # check for python3
    elif sys.version_info >= (3, 0):
        pass
    # check for windows python2. SPL-175233 -- python3 stdout is already binary
    elif sys.platform == 'win32':
        # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
        # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
        # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
        from platform import python_implementation
        implementation = python_implementation()
        if implementation == 'PyPy':
            return os.fdopen(fh.fileno(), 'wb', 0)
        else:
            import msvcrt
            msvcrt.setmode(fh.fileno(), os.O_BINARY)
    return fh 
Example 28
Project: misp42splunk   Author: remg427   File: makefile.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def backport_makefile(
    self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
    """
    Backport of ``socket.makefile`` from Python 3.5.
    """
    if not set(mode) <= {"r", "w", "b"}:
        raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
    writing = "w" in mode
    reading = "r" in mode or not writing
    assert reading or writing
    binary = "b" in mode
    rawmode = ""
    if reading:
        rawmode += "r"
    if writing:
        rawmode += "w"
    raw = SocketIO(self, rawmode)
    self._makefile_refs += 1
    if buffering is None:
        buffering = -1
    if buffering < 0:
        buffering = io.DEFAULT_BUFFER_SIZE
    if buffering == 0:
        if not binary:
            raise ValueError("unbuffered streams must be binary")
        return raw
    if reading and writing:
        buffer = io.BufferedRWPair(raw, raw, buffering)
    elif reading:
        buffer = io.BufferedReader(raw, buffering)
    else:
        assert writing
        buffer = io.BufferedWriter(raw, buffering)
    if binary:
        return buffer
    text = io.TextIOWrapper(buffer, encoding, errors, newline)
    text.mode = mode
    return text 
Example 29
Project: misp42splunk   Author: remg427   File: internals.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def set_binary_mode(fh):
    """ Helper method to set up binary mode for file handles.
    Emphasis being sys.stdin, sys.stdout, sys.stderr.
    For python3, we want to return .buffer
    For python2+windows we want to set os.O_BINARY
    """
    typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
    # check for file handle
    if not isinstance(fh, typefile):
        return fh

    # check for python3 and buffer
    if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
        return fh.buffer
    # check for python3
    elif sys.version_info >= (3, 0):
        pass
    # check for windows python2. SPL-175233 -- python3 stdout is already binary
    elif sys.platform == 'win32':
        # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
        # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
        # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
        from platform import python_implementation
        implementation = python_implementation()
        if implementation == 'PyPy':
            return os.fdopen(fh.fileno(), 'wb', 0)
        else:
            import msvcrt
            msvcrt.setmode(fh.fileno(), os.O_BINARY)
    return fh 
Example 30
Project: misp42splunk   Author: remg427   File: parser.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def parse(self, fp, headersonly=False):
        """Create a message structure from the data in a binary file.

        Reads all the data from the file and returns the root of the message
        structure.  Optional headersonly is a flag specifying whether to stop
        parsing after reading the headers or not.  The default is False,
        meaning it parses the entire contents of the file.
        """
        fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape')
        with fp:
            return self.parser.parse(fp, headersonly)