Python itertools.izip() Examples

def maxSeries(requestContext, *seriesLists):
    """
    Takes one metric or a wildcard seriesList.
    For each datapoint from each metric passed in, pick the maximum value and graph it.

    Example:

    .. code-block:: none

      &target=maxSeries(Server*.connections.total)

    """
    yield defer.succeed(None)
    (seriesList, start, end, step) = normalize(seriesLists)
    name = "maxSeries(%s)" % formatPathExpressions(seriesList)
    values = (safeMax(row) for row in izip(*seriesList))
    series = TimeSeries(name, start, end, step, values)
    series.pathExpression = name
    returnValue([series]) 

def tag(self, data_iter):
        """A tagging function.

        Args:
            data_iter: A iterator for generate batches.

        Returns:
            A generator for tagging result.
        """
        output = []
        for data in data_iter:
            batch = data_to_ids(data, [self.item2id] + [self.word2id] * self.parameters['word_window_size'])
            batch = create_input(batch)
            seq_ids, seq_other_ids_list, seq_lengths = batch[0], batch[1: -1], batch[-1]
            feed_dict = {self.seq_ids_pl: seq_ids.astype(INT_TYPE),
                         self.seq_lengths_pl: seq_lengths.astype(INT_TYPE),
                         self.is_train_pl: False}
            for pl, v in zip(self.seq_other_ids_pls, seq_other_ids_list):
                feed_dict[pl] = v.astype(INT_TYPE)
            scores = self.sess.run(self.scores_op, feed_dict)
            stag_ids = self.inference(scores, seq_lengths)
            for seq, stag_id, length in izip(data[0], stag_ids, seq_lengths):
                output.append((seq, [self.id2tag[t] for t in stag_id[:length]]))
            yield zip(*output)
            output = [] 

def _records_protocol_v1(self, ifile):

        reader = csv.reader(ifile, dialect=CsvDialect)

        try:
            fieldnames = reader.next()
        except StopIteration:
            return

        mv_fieldnames = {name: name[len('__mv_'):] for name in fieldnames if name.startswith('__mv_')}

        if len(mv_fieldnames) == 0:
            for values in reader:
                yield OrderedDict(izip(fieldnames, values))
            return

        for values in reader:
            record = OrderedDict()
            for fieldname, value in izip(fieldnames, values):
                if fieldname.startswith('__mv_'):
                    if len(value) > 0:
                        record[mv_fieldnames[fieldname]] = self._decode_list(value)
                elif fieldname not in record:
                    record[fieldname] = value
            yield record 

def go(fhs):
  fmt = None
  with open(opt_vocab, 'w') as vocab_out:
    with open(opt_output, 'w') as vecs_out:
      for lines in izip(*fhs):
        parts = [line.split() for line in lines]
        token = parts[0][0]
        if any(part[0] != token for part in parts[1:]):
          raise IOError('vector files must be aligned')

        print >> vocab_out, token

        vec = [sum(float(x) for x in xs) for xs in zip(*parts)[1:]]
        if not fmt:
          fmt = struct.Struct('%df' % len(vec))

        vecs_out.write(fmt.pack(*vec)) 

def convert_to_graph_tool(G):
  timer = utils.Timer()
  timer.tic()
  gtG = gt.Graph(directed=G.is_directed())
  gtG.ep['action'] = gtG.new_edge_property('int')

  nodes_list = G.nodes()
  nodes_array = np.array(nodes_list)

  nodes_id = np.zeros((nodes_array.shape[0],), dtype=np.int64)

  for i in range(nodes_array.shape[0]):
    v = gtG.add_vertex()
    nodes_id[i] = int(v)

  # d = {key: value for (key, value) in zip(nodes_list, nodes_id)}
  d = dict(itertools.izip(nodes_list, nodes_id))

  for src, dst, data in G.edges_iter(data=True):
    e = gtG.add_edge(d[src], d[dst])
    gtG.ep['action'][e] = data['action']
  nodes_to_id = d
  timer.toc(average=True, log_at=1, log_str='src.graph_utils.convert_to_graph_tool')
  return gtG, nodes_array, nodes_to_id 

def create_input(batch):
    """
    Take each sentence data in batch and return an input for
    the training or the evaluation function.
    """
    assert len(batch) > 0
    lengths = [len(seq) for seq in batch[0]]
    max_len = max(2, max(lengths))
    ret = []
    for d in batch:
        dd = []
        for seq_id, pos in izip(d, lengths):
            assert len(seq_id) == pos
            pad = [0] * (max_len - pos)
            dd.append(seq_id + pad)
        ret.append(np.array(dd))
    ret.append(np.array(lengths))
    return ret 

def averageSeries(requestContext, *seriesLists):
    """
    Short Alias: avg()

    Takes one metric or a wildcard seriesList.
    Draws the average value of all metrics passed at each time.

    Example:

    .. code-block:: none

      &target=averageSeries(company.server.*.threads.busy)

    """
    yield defer.succeed(None)
    (seriesList, start, end, step) = normalize(seriesLists)
    name = "averageSeries(%s)" % formatPathExpressions(seriesList)
    values = (safeDiv(safeSum(row), safeLen(row)) for row in izip(*seriesList))
    series = TimeSeries(name, start, end, step, values)
    series.pathExpression = name
    returnValue([series]) 

def stddevSeries(requestContext, *seriesLists):
    """

    Takes one metric or a wildcard seriesList.
    Draws the standard deviation of all metrics passed at each time.

    Example:

    .. code-block:: none

      &target=stddevSeries(company.server.*.threads.busy)

    """
    yield defer.succeed(None)
    (seriesList, start, end, step) = normalize(seriesLists)
    name = "stddevSeries(%s)" % formatPathExpressions(seriesList)
    values = (safeStdDev(row) for row in izip(*seriesList))
    series = TimeSeries(name, start, end, step, values)
    series.pathExpression = name
    returnValue([series]) 

def rangeOfSeries(requestContext, *seriesLists):
    """
    Takes a wildcard seriesList.
    Distills down a set of inputs into the range of the series

    Example:

    .. code-block:: none

        &target=rangeOfSeries(Server*.connections.total)

    """
    yield defer.succeed(None)
    (seriesList, start, end, step) = normalize(seriesLists)
    name = "rangeOfSeries(%s)" % formatPathExpressions(seriesList)
    values = (safeSubtract(max(row), min(row)) for row in izip(*seriesList))
    series = TimeSeries(name, start, end, step, values)
    series.pathExpression = name
    returnValue([series]) 

def percentileOfSeries(requestContext, seriesList, n, interpolate=False):
    """
    percentileOfSeries returns a single series which is composed of the n-percentile
    values taken across a wildcard series at each point. Unless `interpolate` is
    set to True, percentile values are actual values contained in one of the
    supplied series.
    """
    yield defer.succeed(None)
    if n <= 0:
        raise ValueError(
            'The requested percent is required to be greater than 0')

    name = 'percentileOfSeries(%s,%g)' % (seriesList[0].pathExpression, n)
    (start, end, step) = normalize([seriesList])[1:]
    values = [_getPercentile(row, n, interpolate) for row in izip(*seriesList)]
    resultSeries = TimeSeries(name, start, end, step, values)
    resultSeries.pathExpression = name

    returnValue([resultSeries]) 

def countSeries(requestContext, *seriesLists):
    """
    Draws a horizontal line representing the number of nodes found in the seriesList.

    .. code-block:: none

      &target=countSeries(carbon.agents.*.*)

    """
    yield defer.succeed(None)
    (seriesList, start, end, step) = normalize(seriesLists)
    name = "countSeries(%s)" % formatPathExpressions(seriesList)
    values = (int(len(row)) for row in izip(*seriesList))
    series = TimeSeries(name, start, end, step, values)
    series.pathExpression = name
    returnValue([series]) 

def constant_time_compare(val1, val2):
    """Returns True if the two strings are equal, False otherwise.

    The time taken is independent of the number of characters that match.  Do
    not use this function for anything else than comparision with known
    length targets.

    This is should be implemented in C in order to get it completely right.
    """
    if _builtin_constant_time_compare is not None:
        return _builtin_constant_time_compare(val1, val2)
    len_eq = len(val1) == len(val2)
    if len_eq:
        result = 0
        left = val1
    else:
        result = 1
        left = val2
    for x, y in izip(bytearray(left), bytearray(val2)):
        result |= x ^ y
    return result == 0 

def _create_config_parameters():
  """Creates a config value pair for parameterized test cases.

  Yields:
    A list containing the list of configs and their values.
  """
  string_config_value = 'config value 1'
  integer_config_value = 1
  bool_config_value = True
  list_config_value = ['email1', 'email2']
  config_ids = ['string_config', 'integer_config', 'bool_config', 'list_config']
  config_values = [
      string_config_value, integer_config_value, bool_config_value,
      list_config_value
  ]
  for i in itertools.izip(config_ids, config_values):
    yield [i] 

def run_train_epoch(X_train, Y_train, batchsize, train_fn):
    total_mmd2 = 0
    total_obj = 0
    n_batches = 0
    batches = itertools.izip( # shuffle the two independently
        iterate_minibatches(X_train, batchsize=batchsize, shuffle=True),
        iterate_minibatches(Y_train, batchsize=batchsize, shuffle=True),
    )
    for ((Xbatch,), (Ybatch,)) in batches:
        mmd2, obj = train_fn(Xbatch, Ybatch)
        assert np.isfinite(mmd2)
        assert np.isfinite(obj)
        total_mmd2 += mmd2
        total_obj += obj
        n_batches += 1
    return total_mmd2 / n_batches, total_obj / n_batches 

def _check_returns_correct_shapes_with_dynamic_inputs(
      self, batch_size, image_height, image_width, depth_multiplier,
      expected_feature_map_shapes, use_explicit_padding=False,
      layer_names=None):
    def graph_fn(image_height, image_width):
      image_tensor = tf.random_uniform([batch_size, image_height, image_width,
                                        3], dtype=tf.float32)
      model = self._create_application_with_layer_outputs(
          layer_names=layer_names,
          batchnorm_training=False, use_explicit_padding=use_explicit_padding,
          alpha=depth_multiplier)
      return model(image_tensor)

    feature_maps = self.execute_cpu(graph_fn, [
        np.array(image_height, dtype=np.int32),
        np.array(image_width, dtype=np.int32)
    ])

    for feature_map, expected_shape in itertools.izip(
        feature_maps, expected_feature_map_shapes):
      self.assertAllEqual(feature_map.shape, expected_shape) 

def _check_returns_correct_shape(
      self, batch_size, image_height, image_width, depth_multiplier,
      expected_feature_map_shapes, use_explicit_padding=False, min_depth=None,
      layer_names=None):
    def graph_fn(image_tensor):
      model = self._create_application_with_layer_outputs(
          layer_names=layer_names,
          batchnorm_training=False, use_explicit_padding=use_explicit_padding,
          min_depth=min_depth,
          alpha=depth_multiplier)
      return model(image_tensor)

    image_tensor = np.random.rand(batch_size, image_height, image_width,
                                  3).astype(np.float32)
    feature_maps = self.execute(graph_fn, [image_tensor])

    for feature_map, expected_shape in itertools.izip(
        feature_maps, expected_feature_map_shapes):
      self.assertAllEqual(feature_map.shape, expected_shape) 

def _calculate_annual_sunlight_exposure(
            values, hoys, threshhold=None, blinds_state_ids=None, occ_schedule=None,
            target_hours=None):
        threshhold = threshhold or 1000
        target_hours = target_hours or 250
        schedule = occ_schedule or Schedule.eight_am_to_six_pm()
        ase = 0
        problematic_hours = []
        for h, v in zip(hoys, values):
            if h not in schedule:
                continue
            if v > threshhold:
                ase += 1
                problematic_hours.append(h)

        return ase < target_hours, ase, problematic_hours 

def check_extract_features_returns_correct_shapes_with_dynamic_inputs(
      self, batch_size, image_height, image_width, depth_multiplier,
      pad_to_multiple, expected_feature_map_shapes, use_explicit_padding=False,
      use_keras=False):
    def graph_fn(image_height, image_width):
      image_tensor = tf.random_uniform([batch_size, image_height, image_width,
                                        3], dtype=tf.float32)
      return self._extract_features(image_tensor,
                                    depth_multiplier,
                                    pad_to_multiple,
                                    use_explicit_padding,
                                    use_keras=use_keras)

    feature_maps = self.execute_cpu(graph_fn, [
        np.array(image_height, dtype=np.int32),
        np.array(image_width, dtype=np.int32)
    ])
    for feature_map, expected_shape in itertools.izip(
        feature_maps, expected_feature_map_shapes):
      self.assertAllEqual(feature_map.shape, expected_shape) 

def check_extract_features_returns_correct_shape(
      self, batch_size, image_height, image_width, depth_multiplier,
      pad_to_multiple, expected_feature_map_shapes, use_explicit_padding=False,
      use_keras=False):
    def graph_fn(image_tensor):
      return self._extract_features(image_tensor,
                                    depth_multiplier,
                                    pad_to_multiple,
                                    use_explicit_padding,
                                    use_keras=use_keras)

    image_tensor = np.random.rand(batch_size, image_height, image_width,
                                  3).astype(np.float32)
    feature_maps = self.execute(graph_fn, [image_tensor])
    for feature_map, expected_shape in itertools.izip(
        feature_maps, expected_feature_map_shapes):
      self.assertAllEqual(feature_map.shape, expected_shape) 

def write_cluster_ids(words, cluster_ids, out=None):
    """Write given list of words and their corresponding cluster ids to out."""

    assert len(words) == len(cluster_ids), 'word/cluster ids number mismatch'

    if out is None:
        out = sys.stdout
    for word, cid in izip(words, cluster_ids):
        print >> out, '%s\t%d' % (word, cid) 

def phase1(self): # Compute common names
        a = dict(izip(imap(os.path.normcase, self.left_list), self.left_list))
        b = dict(izip(imap(os.path.normcase, self.right_list), self.right_list))
        self.common = map(a.__getitem__, ifilter(b.__contains__, a))
        self.left_only = map(a.__getitem__, ifilterfalse(b.__contains__, a))
        self.right_only = map(b.__getitem__, ifilterfalse(a.__contains__, b)) 

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

# transated from http://www.hackersdelight.org/hdcodetxt/snoob.c.txt 

def __iter__(self):
        """Iterate over (word, vector) pairs."""

        #return izip(self.vocab.words(), self._vectors)
        return izip(self.vocab.iterwords(), iter(self._vectors)) 

def write_cluster_ids(words, cluster_ids, out=None):
    """Write given list of words and their corresponding cluster ids to out."""

    assert len(words) == len(cluster_ids), 'word/cluster ids number mismatch'

    if out is None:
        out = sys.stdout
    for word, cid in izip(words, cluster_ids):
        print >> out, '%s\t%d' % (word, cid) 

def nlargest(n, iterable, key=None):
    """Find the n largest elements in a dataset.

    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
    """

    # Short-cut for n==1 is to use max() when len(iterable)>0
    if n == 1:
        it = iter(iterable)
        head = list(islice(it, 1))
        if not head:
            return []
        if key is None:
            return [max(chain(head, it))]
        return [max(chain(head, it), key=key)]

    # When n>=size, it's faster to use sorted()
    try:
        size = len(iterable)
    except (TypeError, AttributeError):
        pass
    else:
        if n >= size:
            return sorted(iterable, key=key, reverse=True)[:n]

    # When key is none, use simpler decoration
    if key is None:
        it = izip(iterable, count(0,-1))                    # decorate
        result = _nlargest(n, it)
        return map(itemgetter(0), result)                   # undecorate

    # General case, slowest method
    in1, in2 = tee(iterable)
    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
    result = _nlargest(n, it)
    return map(itemgetter(2), result)                       # undecorate 

def nsmallest(n, iterable, key=None):
    """Find the n smallest elements in a dataset.

    Equivalent to:  sorted(iterable, key=key)[:n]
    """
    # Short-cut for n==1 is to use min() when len(iterable)>0
    if n == 1:
        it = iter(iterable)
        head = list(islice(it, 1))
        if not head:
            return []
        if key is None:
            return [min(chain(head, it))]
        return [min(chain(head, it), key=key)]

    # When n>=size, it's faster to use sorted()
    try:
        size = len(iterable)
    except (TypeError, AttributeError):
        pass
    else:
        if n >= size:
            return sorted(iterable, key=key)[:n]

    # When key is none, use simpler decoration
    if key is None:
        it = izip(iterable, count())                        # decorate
        result = _nsmallest(n, it)
        return map(itemgetter(0), result)                   # undecorate

    # General case, slowest method
    in1, in2 = tee(iterable)
    it = izip(imap(key, in1), count(), in2)                 # decorate
    result = _nsmallest(n, it)
    return map(itemgetter(2), result)                       # undecorate 

def save_mtx(dir_name, X, genes):
    X = X.tocoo()
    
    if not os.path.exists(dir_name):
        mkdir_p(dir_name)

    with open(dir_name + '/matrix.mtx', 'w') as f:
        f.write('%%MatrixMarket matrix coordinate integer general\n')
        
        f.write('{} {} {}\n'.format(X.shape[1], X.shape[0], X.nnz))

        try:
            from itertools import izip
        except ImportError:
            izip = zip
        
        for i, j, val in izip(X.row, X.col, X.data):
            f.write('{} {} {}\n'.format(j + 1, i + 1, int(val)))

    with open(dir_name + '/genes.tsv', 'w') as f:
        for idx, gene in enumerate(genes):
            f.write('{}\t{}\n'.format(idx + 1, gene))

    with open(dir_name + '/barcodes.tsv', 'w') as f:
        for idx in range(X.shape[0]):
            f.write('cell{}-1\n'.format(idx)) 

def iter_explain(self, instances_df, nh_size):

        [Xs, Ys, isSparse] = self.preprocessor.generate_samples(nh_size)
        [Xe, Ye, isSparse] = self.preprocessor.preprocess(instances_df)

        sample_weights = self.compute_sample_weights_to_instance(Xe, Xs)
        classes = self.preprocessor.get_classes()
        predictor_features = self.preprocessor.get_predictor_features()
        coefs_cols = ['coef_{}'.format(c) for c in classes]
        predictor_features_df = pd.DataFrame(predictor_features, columns=['feature'])
        samples_cols = ['sample_{}'.format(s) for s in range(nh_size)]

        for row_idx, [to_exp, to_proba, w] in enumerate(izip(Xe, Ye, sample_weights)):
            Xs[0,:] = to_exp
            Ys[0,:] = to_proba
            model_regressor = Ridge(alpha=self.ridge_alpha, fit_intercept=True, random_state=self.random_state)
            #TODO: compare with train explanation learning
            model_regressor.fit(Xs,Ys, sample_weight=w)
            local_r2_score = model_regressor.score(Xs, Ys, sample_weight=None)
            intercept_np = model_regressor.intercept_
            model_coefs = model_regressor.coef_
            kernel_distance_avg = np.mean(w)
            kernel_distance_std = np.std(w)

            coefs_df = pd.DataFrame(model_coefs.T, columns=coefs_cols)
            explanation_df = pd.concat((predictor_features_df,coefs_df), axis=1)
            #TODO: optimize this
            explanation_df.insert(0, '_exp_id', row_idx)

            instance_df = pd.DataFrame(to_exp.reshape(-1, len(to_exp)), columns=predictor_features)
            instance_df['r2_score'] = local_r2_score
            instance_df['kernel_distance_avg'] = kernel_distance_avg
            instance_df['kernel_distance_std'] = kernel_distance_std
            #TODO: optimize this
            instance_df.insert(0, '_exp_id', row_idx)

            #FIXME: used only for debugging 
            #weights_df = pd.DataFrame(w.reshape(-1, len(w)), columns=samples_cols)
            #weights_df.insert(0, '_exp_id', row_idx)

            yield explanation_df, instance_df 

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return itertools.izip(a, b) 

def map(self, fn, *iterables, **kwargs):
        """Returns an iterator equivalent to map(fn, iter).

        Args:
            fn: A callable that will take as many arguments as there are
                passed iterables.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.

        Returns:
            An iterator equivalent to: map(func, *iterables) but the calls may
            be evaluated out-of-order.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
            Exception: If fn(*args) raises for any values.
        """
        timeout = kwargs.get('timeout')
        if timeout is not None:
            end_time = timeout + time.time()

        fs = [self.submit(fn, *args) for args in itertools.izip(*iterables)]

        # Yield must be hidden in closure so that the futures are submitted
        # before the first iterator value is required.
        def result_iterator():
            try:
                # reverse to keep finishing order
                fs.reverse()
                while fs:
                    # Careful not to keep a reference to the popped future
                    if timeout is None:
                        yield fs.pop().result()
                    else:
                        yield fs.pop().result(end_time - time.time())
            finally:
                for future in fs:
                    future.cancel()
        return result_iterator()