Python itertools.tee() Examples

def test_peek(self):

        def give_gen():
            for i in range(1, 5):
                yield i

        def give_empty_gen():
            for i in range(1, 1):
                yield i

        obj = give_gen()

        for i in range(1, 5):
            num, new_obj = peek(obj)
            obj, new_obj = itertools.tee(obj)
            self.assertEqual(i, num)

        ret_val = peek(obj)
        obj = give_empty_gen()
        ret_val_1 = peek(obj)

        self.assertEqual(ret_val, None)
        self.assertEqual(ret_val_1, None) 

def build_dataset(args):

    print("Building dataset from : {}".format(args.input))
    print("-> Building {} random splits".format(args.nb_splits))

    nlp = spacy.load('en', create_pipeline=custom_pipeline)
    gen_a,gen_b = itertools.tee(data_generator(args.input),2)
    data = [(z["reviewerID"],z["asin"],tok,z["overall"]) for z,tok in zip(tqdm((z for z in gen_a),desc="reading file"),nlp.pipe((x["reviewText"] for x in gen_b), batch_size=1000000, n_threads=8))]

    print(data[0])
    shuffle(data)

    splits = [randint(0,args.nb_splits-1) for _ in range(0,len(data))]
    count = Counter(splits)

    print("Split distribution is the following:")
    print(count)

    return {"data":data,"splits":splits,"rows":("user_id","item_id","review","rating")} 

def iterintervals(self, n=2):
        """Iterate over groups of `n` consecutive measurement points in the
        time series.

        """
        # tee the original iterator into n identical iterators
        streams = itertools.tee(iter(self), n)

        # advance the "cursor" on each iterator by an increasing
        # offset, e.g. if n=3:
        #
        #                   [a, b, c, d, e, f, ..., w, x, y, z]
        #  first cursor -->  *
        # second cursor -->     *
        #  third cursor -->        *
        for stream_index, stream in enumerate(streams):
            for _ in range(stream_index):
                next(stream)

        # now, zip the offset streams back together to yield tuples,
        # in the n=3 example it would yield:
        # (a, b, c), (b, c, d), ..., (w, x, y), (x, y, z)
        for intervals in zip(*streams):
            yield intervals 

def sliding_window(n, seq):
    """
    Copied from toolz
    https://toolz.readthedocs.io/en/latest/_modules/toolz/itertoolz.html#sliding_window

    A sequence of overlapping subsequences

    >>> list(sliding_window(2, [1, 2, 3, 4]))
    [(1, 2), (2, 3), (3, 4)]

    This function creates a sliding window suitable for transformations like
    sliding means / smoothing

    >>> mean = lambda seq: float(sum(seq)) / len(seq)
    >>> list(map(mean, sliding_window(2, [1, 2, 3, 4])))
    [1.5, 2.5, 3.5]
    """
    return zip(*(collections.deque(itertools.islice(it, i), 0) or it
                 for i, it in enumerate(itertools.tee(seq, n)))) 

def follow_path(self, path, types, call_scope):
        """
        Follows a path like::

            self.follow_path(iter(['Foo', 'bar']), [a_type], from_somewhere)

        to follow a call like ``module.a_type.Foo.bar`` (in ``from_somewhere``).
        """
        results_new = []
        iter_paths = itertools.tee(path, len(types))

        for i, typ in enumerate(types):
            fp = self._follow_path(iter_paths[i], typ, call_scope)
            if fp is not None:
                results_new += fp
            else:
                # This means stop iteration.
                return types
        return results_new 

def unnest(item):
    # type: (Any) -> Iterable[Any]
    target = None  # type: Optional[Iterable]
    if isinstance(item, Iterable) and not isinstance(item, six.string_types):
        item, target = itertools.tee(item, 2)
    else:
        target = item
    if getattr(target, "__iter__", None):
        for el in target:
            if isinstance(el, Iterable) and not isinstance(el, six.string_types):
                el, el_copy = itertools.tee(el, 2)
                for sub in unnest(el_copy):
                    yield sub
            else:
                yield el
    else:
        yield target 

def __call__(self, jobs_or_statepoints):
        "Evaluate the schema for the given state points."
        s = dict()
        iterators = itertools.tee(jobs_or_statepoints, len(self))
        for key, it in zip(self, iterators):
            values = []
            keys = key.split('.')
            for sp in it:
                if not isinstance(sp, Mapping):
                    sp = sp.statepoint
                v = sp[keys[0]]
                for k in keys[1:]:
                    v = v[k]
                values.append(v)
            s[key] = _collect_by_type(values)
        return ProjectSchema(s) 

def random_bipartition_iter(iterable, relative_size=.5, random_state=1):
    """random_bipartition_iter."""
    size_iterable, iterable1, iterable2 = tee(iterable, 3)
    size = iterator_size(size_iterable)
    part1_ids, part2_ids = random_bipartition(
        size, relative_size=relative_size, random_state=random_state)
    part1_iterable = selection_iterator(iterable1, part1_ids)
    part2_iterable = selection_iterator(iterable2, part2_ids)
    return part1_iterable, part2_iterable 

def random_bipartition_iter(iterable, relative_size=.5, random_state=1):
    """random_bipartition_iter."""
    size_iterable, iterable1, iterable2 = tee(iterable, 3)
    size = iterator_size(size_iterable)
    part1_ids, part2_ids = random_bipartition(
        size, relative_size=relative_size, random_state=random_state)
    part1_iterable = selection_iterator(iterable1, part1_ids)
    part2_iterable = selection_iterator(iterable2, part2_ids)
    return part1_iterable, part2_iterable 

def join_pre_processes(iterable, pre_processes=None, weights=None):
    """join_pre_processes."""
    graphs_list = list()
    assert(len(weights) == len(pre_processes)), 'Different lengths'
    # NOTE: we have to duplicate the sequences iterator if we want to use
    # different modifiers in parallel
    iterables = tee(iterable, len(pre_processes))
    for pre_process_item, iterable_item in zip(pre_processes, iterables):
        graphs_list.append(pre_process_item(iterable_item))
    return (graphs_list, weights) 

def auto_relabel(graphs, n_clusters=16, **opts):
    """Label nodes with cluster id."""
    graphs, graphs_ = tee(graphs)
    label_list, vecs_list = auto_label(graphs_, n_clusters=n_clusters, **opts)
    relabeled_graphs = []
    for labels, vecs, orig_graph in zip(label_list, vecs_list, graphs):
        graph = nx.Graph(orig_graph)
        for label, vec, u in zip(labels, vecs, graph.nodes()):
            graph.nodes[u]['label'] = label
            graph.nodes[u]['vec'] = list(vec)
        relabeled_graphs.append(graph)
    return relabeled_graphs 

def shared_parent(self):
        """
        Returns the folder object that is the shared parent (the root of
        a shared folder hierarchy) or None if there is no shared parent.
        """
        root = self
        a, b = itertools.tee(reversed(self.breadcrumbs()))
        next(b, None)
        for folder, parent in itertools.zip_longest(a, b):
            if folder.shared:
                root = folder
            if parent is None or not parent.shared:
                break
        return root 

def nsmallest(n, iterable, key=None):
    """Find the n smallest elements in a dataset.

    Equivalent to:  sorted(iterable, key=key)[:n]
    """
    # Short-cut for n==1 is to use min() when len(iterable)>0
    if n == 1:
        it = iter(iterable)
        head = list(islice(it, 1))
        if not head:
            return []
        if key is None:
            return [min(chain(head, it))]
        return [min(chain(head, it), key=key)]

    # When n>=size, it's faster to use sorted()
    try:
        size = len(iterable)
    except (TypeError, AttributeError):
        pass
    else:
        if n >= size:
            return sorted(iterable, key=key)[:n]

    # When key is none, use simpler decoration
    if key is None:
        it = zip(iterable, count())                         # decorate
        result = _nsmallest(n, it)
        return [r[0] for r in result]                       # undecorate

    # General case, slowest method
    in1, in2 = tee(iterable)
    it = zip(map(key, in1), count(), in2)                   # decorate
    result = _nsmallest(n, it)
    return [r[2] for r in result]                           # undecorate 

def nlargest(n, iterable, key=None):
    """Find the n largest elements in a dataset.

    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
    """

    # Short-cut for n==1 is to use max() when len(iterable)>0
    if n == 1:
        it = iter(iterable)
        head = list(islice(it, 1))
        if not head:
            return []
        if key is None:
            return [max(chain(head, it))]
        return [max(chain(head, it), key=key)]

    # When n>=size, it's faster to use sorted()
    try:
        size = len(iterable)
    except (TypeError, AttributeError):
        pass
    else:
        if n >= size:
            return sorted(iterable, key=key, reverse=True)[:n]

    # When key is none, use simpler decoration
    if key is None:
        it = zip(iterable, count(0,-1))                     # decorate
        result = _nlargest(n, it)
        return [r[0] for r in result]                       # undecorate

    # General case, slowest method
    in1, in2 = tee(iterable)
    it = zip(map(key, in1), count(0,-1), in2)               # decorate
    result = _nlargest(n, it)
    return [r[2] for r in result]                           # undecorate 

def __iter__(self) -> Iterator[Any]:
        if self._computed:
            yield from self._dataset
        else:
            iterable, self._iterable = tee(self._iterable)
            yield from iterable 

def test_search(self):
        actual = self.pclient.images.search("alpine", 25)
        names, length = itertools.tee(actual)

        for img in names:
            self.assertIn("alpine", img.name)
        self.assertTrue(0 < len(list(length)) <= 25) 

def drop(self, project):
        self.redis.srem(self.__prefix__ + 'projects', project)

        if self.scan_available:
            scan_method = self.redis.scan_iter
        else:
            scan_method = self.redis.keys

        for each in itertools.tee(scan_method("%s%s_*" % (self.__prefix__, project)), 100):
            each = list(each)
            if each:
                self.redis.delete(*each) 

def tee(self, tee, *args, **kwargs):
        if not isinstance(tee, TeePipe):
            tee = TeePipe(tee, *args, **kwargs)
        return self.__ins_add(self._pipeline, tee, TransformType.TEE, args, kwargs) 

def __check_pipeline(self):
        '''
        Ensure that pipeline runnable and behaves as expected. Currently checks
        that:
            - Clustering operations are followed by a nested transform or an
              aggregation. If not, an explicity generator -> list mapping operation
              is added to avoid subsequent operations being confused by the
              generator of generators that clustering operations yield.
            - If the last transformation is a clustering operation, that the
              same explicity mapping is done to ensure sensible output from the
              transform pipe.
        '''

        def pairwise(iterable):
            "s -> (s0,s1), (s1,s2), (s2, s3), ..."
            a, b = tee(iterable)
            next(b, None)
            return zip(a, b)

        pipeline = self._pipeline[:1]

        for a, b in pairwise(self._pipeline):
            if a.type == TransformType.CLUSTER and b.type not in (TransformType.NESTED, TransformType.AGGREGATE):
                logging.warning("A non-nested non-aggregation transform has been used after a clustering pipe. Adding a potentially memory inefficient pipe segment to convert cluster generator to list.")
                pipeline.append(self.__ins(lambda x: list(x), TransformType.MAP, [], {}))
            pipeline.append(b)

        if pipeline[-1].type == TransformType.CLUSTER:
            pipeline.append(self.__ins(lambda x: list(x), TransformType.MAP, [], {}))

        return pipeline 

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

# transated from http://www.hackersdelight.org/hdcodetxt/snoob.c.txt 

def __enter__(self):  # type: () -> None
        # Entering this context manager - save the state
        if PY2:
            # Python 2.7 does not allow to directly copy
            # an iterator, so we have to make tees of the original
            # chars iterator.
            self._source._chars, self._chars = itertools.tee(self._source._chars)
        else:
            self._chars = copy(self._source._chars)
        self._idx = self._source._idx
        self._current = self._source._current
        self._marker = self._source._marker

        return self 

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip(a, b) 

def partition(pred, iterable):
    """Use a predicate to partition entries into false entries and true entries."""
    # https://stackoverflow.com/questions/8793772/how-to-split-a-sequence-according-to-a-predicate
    # NOTE: this might iterate over the collection twice
    # NOTE: need to use filter(s) here because we're lazily dealing with iterators
    it1, it2 = itertools.tee(iterable)
    return itertools.filterfalse(pred, it1), filter(pred, it2)  # pylint: disable=bad-builtin 

def pairwise(iterable, default_value):
    """Return pairs of items from `iterable`.

    pairwise([1, 2, 3], default_value=None) -> (1, 2) (2, 3), (3, None)
    """
    a, b = tee(iterable)
    _ = next(b, default_value)
    return zip_longest(a, b, fillvalue=default_value) 

def parse(self, file, boundary, content_length):
        formstream, filestream = tee(
            self.parse_parts(file, boundary, content_length), 2
        )
        form = (p[1] for p in formstream if p[0] == "form")
        files = (p[1] for p in filestream if p[0] == "file")
        return self.cls(form), self.cls(files) 

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return itertools.izip(a, b) 

def prev_this_next(it, sentinel=None):
    """Utility to return (prev, this, next) tuples from an iterator"""
    i1, i2, i3 = tee(it, 3)
    next(i3, None)
    return zip(chain([sentinel], i1), i2, chain(i3, [sentinel])) 

def ifetch_multiple(iterable, *keys, defaults=None, getter=None):
    defaults = defaults or {}
    if len(keys) > 1:
        iters = tee(iterable, len(keys))
    else:
        iters = (iterable,)
    iters = [ifetch_single(it, key, default=defaults.get(key, EMPTY), getter=getter) for it, key in zip(iters, keys)]
    return iters 

def parse(self, file, boundary, content_length):
        formstream, filestream = tee(
            self.parse_parts(file, boundary, content_length), 2)
        form = (p[1] for p in formstream if p[0] == 'form')
        files = (p[1] for p in filestream if p[0] == 'file')
        return self.cls(form), self.cls(files) 

def _parse_reply(self):
        super()._parse_reply()
        num_replies = unpack_uint(self.data)
        offset_data = self.data[2:2 + 2 * num_replies]
        offsets = (unpack_uint(offset_data[i:i+2]) for i in range(0, len(offset_data), 2))
        start, end = tee(offsets)  # split offsets into start/end indexes
        next(end)   # advance end by 1 so 2nd item is the end index for the first item
        reply_data = [self.data[i:j] for i, j in zip_longest(start, end)]
        values = []

        for data, tag in zip(reply_data, self.tags):
            service = unpack_uint(data)
            service_status = data[2]
            tag['service_status'] = service_status
            if service_status != SUCCESS:
                tag['error'] = f'{get_service_status(service_status)} - {get_extended_status(data, 2)}'

            if service == TAG_SERVICES_REPLY['Read Tag']:
                if service_status == SUCCESS:
                    value, dt = parse_read_reply(data[4:], tag['tag_info'], tag['elements'])
                else:
                    value, dt = None, None

                values.append(value)
                tag['value'] = value
                tag['data_type'] = dt

        self.values = values