Python numpy.fromiter() Examples

def create_array(width, height, background=None):
    """returns an array.array or numpy.array of the correct size and
    with the given background color"""
    if numpy is not None:
        if background is None:
            return numpy.zeros(width * height, dtype=numpy.uint32)
        else:
            iterable = (background for _ in range(width * height))
            return numpy.fromiter(iterable, numpy.uint32)
    else:
        # Use the smallest typecode that can store a 32-bit unsigned integer
        typecode = "I" if array.array("I").itemsize >= 4 else "L"
        background = (background if background is not None else
                      ColorForName["transparent"])
        return array.array(typecode, [background] * width * height)


# Taken from rgb.txt and converted to ARGB (with the addition of
# transparent). Default is solid black. 

def make2d(array, cols=None, dtype=None):
    '''
    Make a 2D array from an array of arrays.  The `cols' and `dtype'
    arguments can be omitted if the array is not empty.

    '''
    if (cols is None or dtype is None) and not len(array):
        raise RuntimeError("cols and dtype must be specified for empty "
                           "array")

    if cols is None:
        cols = len(array[0])

    if dtype is None:
        dtype = array[0].dtype

    return _np.fromiter(array, [('_', dtype, (cols,))],
                        count=len(array))['_'] 

def test_constructor_object_dtype(self):
        # GH 11856
        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object)
        assert arr.dtype == SparseDtype(np.object)
        assert np.isnan(arr.fill_value)

        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object,
                          fill_value='A')
        assert arr.dtype == SparseDtype(np.object, 'A')
        assert arr.fill_value == 'A'

        # GH 17574
        data = [False, 0, 100.0, 0.0]
        arr = SparseArray(data, dtype=np.object, fill_value=False)
        assert arr.dtype == SparseDtype(np.object, False)
        assert arr.fill_value is False
        arr_expected = np.array(data, dtype=np.object)
        it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected))
        assert np.fromiter(it, dtype=np.bool).all() 

def decons_obs_group_ids(comp_ids, obs_ids, shape, labels, xnull):
    """
    reconstruct labels from observed group ids

    Parameters
    ----------
    xnull: boolean,
        if nulls are excluded; i.e. -1 labels are passed through
    """

    if not xnull:
        lift = np.fromiter(((a == -1).any() for a in labels), dtype='i8')
        shape = np.asarray(shape, dtype='i8') + lift

    if not is_int64_overflow_possible(shape):
        # obs ids are deconstructable! take the fast route!
        out = decons_group_index(obs_ids, shape)
        return out if xnull or not lift.any() \
            else [x - y for x, y in zip(out, lift)]

    i = unique_label_indices(comp_ids)
    i8copy = lambda a: a.astype('i8', subok=False, copy=True)
    return [i8copy(lab[i]) for lab in labels] 

def divide_qualities_into_bins(qualities, n_bins=4):
    """Divides the raw quality scores in bins according to the mean phred
    quality of the sequence they come from

    Args:
        qualities (list): raw count of all the phred scores and mean sequence
            quality
        n_bins (int): number of bins to create (default: 4)

    Returns:
        list: a list of lists containing the binned quality scores
    """
    logger = logging.getLogger(__name__)
    logger.debug('Dividing qualities into mean clusters')
    bin_lists = [[] for _ in range(n_bins)]  # create list of `n_bins` list
    ranges = np.split(np.array(range(40)), n_bins)
    for quality in qualities:
        mean = int(quality[0][1])  # mean is at 1 and same regardless of b pos
        which_array = 0
        for array in ranges:
            if mean in array:
                read = np.fromiter((q[0] for q in quality), dtype=np.float)
                bin_lists[which_array].append(read)
            which_array += 1
    return bin_lists 

def make2d(array, cols=None, dtype=None):
    '''
    Make a 2D array from an array of arrays.  The `cols' and `dtype'
    arguments can be omitted if the array is not empty.
    '''
    if (cols is None or dtype is None) and not len(array):
        raise RuntimeError("cols and dtype must be specified for empty "
                           "array")

    if cols is None:
        cols = len(array[0])

    if dtype is None:
        dtype = array[0].dtype

    return _np.fromiter(array, [('_', dtype, (cols,))],
                        count=len(array))['_'] 

def test_constructor_object_dtype(self):
        # GH 11856
        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object)
        assert arr.dtype == np.object
        assert np.isnan(arr.fill_value)

        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object,
                          fill_value='A')
        assert arr.dtype == np.object
        assert arr.fill_value == 'A'

        # GH 17574
        data = [False, 0, 100.0, 0.0]
        arr = SparseArray(data, dtype=np.object, fill_value=False)
        assert arr.dtype == np.object
        assert arr.fill_value is False
        arr_expected = np.array(data, dtype=np.object)
        it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected))
        assert np.fromiter(it, dtype=np.bool).all() 

def decons_obs_group_ids(comp_ids, obs_ids, shape, labels, xnull):
    """
    reconstruct labels from observed group ids

    Parameters
    ----------
    xnull: boolean,
        if nulls are excluded; i.e. -1 labels are passed through
    """

    if not xnull:
        lift = np.fromiter(((a == -1).any() for a in labels), dtype='i8')
        shape = np.asarray(shape, dtype='i8') + lift

    if not is_int64_overflow_possible(shape):
        # obs ids are deconstructable! take the fast route!
        out = decons_group_index(obs_ids, shape)
        return out if xnull or not lift.any() \
            else [x - y for x, y in zip(out, lift)]

    i = unique_label_indices(comp_ids)
    i8copy = lambda a: a.astype('i8', subok=False, copy=True)
    return [i8copy(lab[i]) for lab in labels] 

def load_word2vec(word2vec_model_path,embed_size):
    """
    load pretrained word2vec in txt format
    :param word2vec_model_path:
    :return: word2vec_dict. word2vec_dict[word]=vector
    """
    #word2vec_object = codecs.open(word2vec_model_path,'r','utf-8') #open(word2vec_model_path,'r')
    #lines=word2vec_object.readlines()
    #word2vec_dict={}
    #for i,line in enumerate(lines):
    #    if i==0: continue
    #    string_list=line.strip().split(" ")
    #    word=string_list[0]
    #    vector=string_list[1:][0:embed_size]
    #    word2vec_dict[word]=vector
    ######################
    word2vec_dict = {}
    with open(word2vec_model_path, errors='ignore') as f:
        meta = f.readline()
        for line in f.readlines():
            items = line.split(' ')
            #if len(items[0]) > 1 and items[0] in vocab:
            word2vec_dict[items[0]] = np.fromiter(items[1:][0:embed_size], dtype=float)
    return word2vec_dict 

def cartesian_product(X):
    '''
    Numpy version of itertools.product or pandas.compat.product.
    Sometimes faster (for large inputs)...

    Examples
    --------
    >>> cartesian_product([list('ABC'), [1, 2]])
    [array(['A', 'A', 'B', 'B', 'C', 'C'], dtype='|S1'),
 	array([1, 2, 1, 2, 1, 2])]

    '''

    lenX = np.fromiter((len(x) for x in X), dtype=int)
    cumprodX = np.cumproduct(lenX)

    a = np.roll(cumprodX, 1)
    a[0] = 1

    b = cumprodX[-1] / cumprodX

    return [np.tile(np.repeat(x, b[i]), 
                    np.product(a[i]))
               for i, x in enumerate(X)] 

def __load(fh):
        return numpy.fromiter(Obj.__read(fh), dtype=Obj.obj_dtype) 

def __load_ascii(fh, header):
        return numpy.fromiter(Stl.__ascii_reader(fh, header), dtype=Stl.stl_dtype) 

def to_timestamps(values):
    try:
        if len(values) == 0:
            return []
        if isinstance(values[0], (numpy.datetime64, datetime.datetime)):
            times = numpy.array(values)
        else:
            try:
                # Try to convert to float. If it works, then we consider
                # timestamps to be number of seconds since Epoch
                # e.g. 123456 or 129491.1293
                float(values[0])
            except ValueError:
                try:
                    # Try to parse the value as a string of ISO timestamp
                    # e.g. 2017-10-09T23:23:12.123
                    numpy.datetime64(values[0])
                except ValueError:
                    # Last chance: it can be relative timestamp, so convert
                    # to timedelta relative to now()
                    # e.g. "-10 seconds" or "5 minutes"
                    times = numpy.fromiter(
                        numpy.add(numpy.datetime64(utcnow()),
                                  [to_timespan(v, True) for v in values]),
                        dtype='datetime64[ns]', count=len(values))
                else:
                    times = numpy.array(values, dtype='datetime64[ns]')
            else:
                times = numpy.array(values, dtype='float') * 10e8
    except ValueError:
        raise ValueError("Unable to convert timestamps")

    times = times.astype('datetime64[ns]')

    if (times < unix_universal_start64).any():
        raise ValueError('Timestamp must be after Epoch')

    return times 

def _encode_measures(self, measures):
        return numpy.fromiter(measures,
                              dtype=TIMESERIES_ARRAY_DTYPE).tobytes() 

def _transitions_matrix(self):
        """ Return a matrix of transition log probabilities. """
        trans_iter = (self._transitions[sj].logprob(si)
                      for sj in self._states
                      for si in self._states)

        transitions_logprob = np.fromiter(trans_iter, dtype=np.float64)
        N = len(self._states)
        return transitions_logprob.reshape((N, N)).T 

def _outputs_vector(self, symbol):
        """
        Return a vector with log probabilities of emitting a symbol
        when entering states.
        """
        out_iter = (self._output_logprob(sj, symbol) for sj in self._states)
        return np.fromiter(out_iter, dtype=np.float64) 

def _numpy_array_from_2arrays(array1, array2, dtype=numpy.uint16):
    ''' Efficiently combine two 1-D arrays into a single 2-D array.

    Avoids large memory usage by creating the array using
    ``numpy.fromiter`` and then reshaping a view of the resulting
    record array.  This does the equivalent of:

        numpy.array(zip(array1, array2))

    but avoids holding a potentially large number of tuples in memory.

    >>> a = numpy.array([1, 2, 3])
    >>> b = numpy.array([4, 5, 6])
    >>> X = _numpy_array_from_2arrays(a, b)
    >>> X
    array([[1, 4],
           [2, 5],
           [3, 6]], dtype=uint16)

    :param array array1: A 1-D numpy array.
    :param array array2: A second 1-D numpy array.
    :param data-type dtype: Data type for array elements
        (must be same for both arrays)

    :returns: A 2D numpy array combining the two input arrays
    '''
    array_dtype = [('x', dtype), ('y', dtype)]

    return numpy.fromiter(itertools.izip(array1, array2), dtype=array_dtype) \
                .view(dtype=dtype) \
                .reshape((-1, 2)) 

def _get_binary_rewards(self, decisions: np.ndarray, rewards: np.ndarray):

        # If a binarizer function is given and binarization has not taken place already in a neighborhood policy
        if self.binarizer and not self.is_contextual_binarized:
            return np.fromiter((self.binarizer(decisions[index], value)  # convert every decision-reward pair to binary
                                for index, value in enumerate(rewards)), rewards.dtype)
        else:
            return rewards 

def test_types(self):
        ai32 = np.fromiter(self.makegen(), np.int32)
        ai64 = np.fromiter(self.makegen(), np.int64)
        af = np.fromiter(self.makegen(), float)
        assert_(ai32.dtype == np.dtype(np.int32))
        assert_(ai64.dtype == np.dtype(np.int64))
        assert_(af.dtype == np.dtype(float)) 

def test_lengths(self):
        expected = np.array(list(self.makegen()))
        a = np.fromiter(self.makegen(), int)
        a20 = np.fromiter(self.makegen(), int, 20)
        assert_(len(a) == len(expected))
        assert_(len(a20) == 20)
        assert_raises(ValueError, np.fromiter,
                          self.makegen(), int, len(expected) + 10) 

def test_values(self):
        expected = np.array(list(self.makegen()))
        a = np.fromiter(self.makegen(), int)
        a20 = np.fromiter(self.makegen(), int, 20)
        assert_(np.alltrue(a == expected, axis=0))
        assert_(np.alltrue(a20 == expected[:20], axis=0)) 

def test_2592(self):
        # Test iteration exceptions are correctly raised.
        count, eindex = 10, 5
        assert_raises(NIterError, np.fromiter,
                          self.load_data(count, eindex), dtype=int, count=count) 

def test_2592_edge(self):
        # Test iter. exceptions, edge case (exception at end of iterator).
        count = 10
        eindex = count-1
        assert_raises(NIterError, np.fromiter,
                          self.load_data(count, eindex), dtype=int, count=count) 

def test_mem_fromiter_invalid_dtype_string(self):
        x = [1, 2, 3]
        assert_raises(ValueError,
                              np.fromiter, [xi for xi in x], dtype='S') 

def test_fromiter_bytes(self):
        # Ticket #1058
        a = np.fromiter(list(range(10)), dtype='b')
        b = np.fromiter(list(range(10)), dtype='B')
        assert_(np.alltrue(a == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])))
        assert_(np.alltrue(b == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))) 

def test_fromiter_comparison(self):
        a = np.fromiter(list(range(10)), dtype='b')
        b = np.fromiter(list(range(10)), dtype='B')
        assert_(np.alltrue(a == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])))
        assert_(np.alltrue(b == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))) 

def test_duplicate_field_names_assign(self):
        ra = np.fromiter(((i*3, i*2) for i in range(10)), dtype='i8,f8')
        ra.dtype.names = ('f1', 'f2')
        repr(ra)  # should not cause a segmentation fault
        assert_raises(ValueError, setattr, ra.dtype, 'names', ('f1', 'f1')) 

def _prepare_data(self):
        data = self.data
        typlist = self.typlist
        convert_dates = self._convert_dates
        # 1. Convert dates
        if self._convert_dates is not None:
            for i, col in enumerate(data):
                if i in convert_dates:
                    data[col] = _datetime_to_stata_elapsed_vec(data[col],
                                                               self.fmtlist[i])
        # 2. Convert strls
        data = self._convert_strls(data)

        # 3. Convert bad string data to '' and pad to correct length
        dtypes = []
        data_cols = []
        has_strings = False
        native_byteorder = self._byteorder == _set_endianness(sys.byteorder)
        for i, col in enumerate(data):
            typ = typlist[i]
            if typ <= self._max_string_length:
                has_strings = True
                data[col] = data[col].fillna('').apply(_pad_bytes, args=(typ,))
                stype = 'S{type}'.format(type=typ)
                dtypes.append(('c' + str(i), stype))
                string = data[col].str.encode(self._encoding)
                data_cols.append(string.values.astype(stype))
            else:
                values = data[col].values
                dtype = data[col].dtype
                if not native_byteorder:
                    dtype = dtype.newbyteorder(self._byteorder)
                dtypes.append(('c' + str(i), dtype))
                data_cols.append(values)
        dtypes = np.dtype(dtypes)

        if has_strings or not native_byteorder:
            self.data = np.fromiter(zip(*data_cols), dtype=dtypes)
        else:
            self.data = data.to_records(index=False) 

def __neighbors_to_graph__(self, neighbors, neighbor_weights,
                               normalization='row', device='cpu'):
        r"""Combine a list of neighbors and neighbor weights to create a sparse
        graph.

        Args:
            neighbors (List[List[int]]): List of neighbors for each node.
            neighbor_weights (List[List[float]]): List of weights for the
                neighbors of each node.
            normalization (str): Normalization of resulting matrix
                (options: :obj:`"row"`, :obj:`"col"`). (default: :obj:`"row"`)
            device (torch.device): Device to create output tensors on.
                (default: :obj:`"cpu"`)

        :rtype: (:class:`LongTensor`, :class:`Tensor`)
        """
        edge_weight = torch.Tensor(np.concatenate(neighbor_weights)).to(device)
        i = np.repeat(np.arange(len(neighbors)),
                      np.fromiter(map(len, neighbors), dtype=np.int))
        j = np.concatenate(neighbors)
        if normalization == 'col':
            edge_index = torch.Tensor(np.vstack([j, i])).to(device)
            N = len(neighbors)
            edge_index, edge_weight = coalesce(edge_index, edge_weight, N, N)
        elif normalization == 'row':
            edge_index = torch.Tensor(np.vstack([i, j])).to(device)
        else:
            raise ValueError(
                f"PPR matrix normalization {normalization} unknown.")
        return edge_index, edge_weight 

def extract_trace_headers(reader, fields, trace_indexes=None):
    """Extract trace header fields from the specified trace headers as separate arrays.

    Args:
        reader: A SegYReader

        fields: A an iterable series where each item is either the name of a field as a string
            or an object such as a NamedField with a 'name' attribute which in turn is the name
            of a field as a string, such as a NamedField.

        trace_indexes: An optional iterable series of trace_indexes. If not provided or None,
            the headers for all trace indexes will be returned.

    Returns:
        A namedtuple with attributes which are one-dimensionsal Numpy arrays.
    """
    if trace_indexes is None:
        trace_indexes = reader.trace_indexes()

    field_names = [_extract_field_name(field) for field in fields]

    class SubFormat(metaclass=SubFormatMeta,
                    parent_format=reader.trace_header_format_class,
                    parent_field_names=field_names):
        pass

    sub_header_packer = make_header_packer(SubFormat, reader.endian)
    trace_header_arrays_cls = namedtuple('trace_header_arrays_cls', field_names)

    trace_headers = [reader.trace_header(trace_index, sub_header_packer)
                     for trace_index in trace_indexes]

    trace_header_arrays = trace_header_arrays_cls(
        *(np.fromiter((getattr(trace_header, field_name) for trace_header in trace_headers),
                      dtype=make_dtype(getattr(SubFormat, field_name).value_type.SEG_Y_TYPE),
                      count=len(trace_headers))
          for field_name in field_names)
    )

    return trace_header_arrays