Python numpy.fromiter() Examples

The following are 30 code examples for showing how to use numpy.fromiter(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module numpy , or try the search function .

Example 1
Project: python-in-practice   Author: lovexiaov   File: __init__.py    License: GNU General Public License v3.0 8 votes vote down vote up
def create_array(width, height, background=None):
    """returns an array.array or numpy.array of the correct size and
    with the given background color"""
    if numpy is not None:
        if background is None:
            return numpy.zeros(width * height, dtype=numpy.uint32)
        else:
            iterable = (background for _ in range(width * height))
            return numpy.fromiter(iterable, numpy.uint32)
    else:
        # Use the smallest typecode that can store a 32-bit unsigned integer
        typecode = "I" if array.array("I").itemsize >= 4 else "L"
        background = (background if background is not None else
                      ColorForName["transparent"])
        return array.array(typecode, [background] * width * height)


# Taken from rgb.txt and converted to ARGB (with the addition of
# transparent). Default is solid black. 
Example 2
Project: pointnet-registration-framework   Author: vinits5   File: plyfile.py    License: MIT License 7 votes vote down vote up
def make2d(array, cols=None, dtype=None):
    '''
    Make a 2D array from an array of arrays.  The `cols' and `dtype'
    arguments can be omitted if the array is not empty.

    '''
    if (cols is None or dtype is None) and not len(array):
        raise RuntimeError("cols and dtype must be specified for empty "
                           "array")

    if cols is None:
        cols = len(array[0])

    if dtype is None:
        dtype = array[0].dtype

    return _np.fromiter(array, [('_', dtype, (cols,))],
                        count=len(array))['_'] 
Example 3
Project: recruit   Author: Frank-qlu   File: test_array.py    License: Apache License 2.0 6 votes vote down vote up
def test_constructor_object_dtype(self):
        # GH 11856
        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object)
        assert arr.dtype == SparseDtype(np.object)
        assert np.isnan(arr.fill_value)

        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object,
                          fill_value='A')
        assert arr.dtype == SparseDtype(np.object, 'A')
        assert arr.fill_value == 'A'

        # GH 17574
        data = [False, 0, 100.0, 0.0]
        arr = SparseArray(data, dtype=np.object, fill_value=False)
        assert arr.dtype == SparseDtype(np.object, False)
        assert arr.fill_value is False
        arr_expected = np.array(data, dtype=np.object)
        it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected))
        assert np.fromiter(it, dtype=np.bool).all() 
Example 4
Project: recruit   Author: Frank-qlu   File: sorting.py    License: Apache License 2.0 6 votes vote down vote up
def decons_obs_group_ids(comp_ids, obs_ids, shape, labels, xnull):
    """
    reconstruct labels from observed group ids

    Parameters
    ----------
    xnull: boolean,
        if nulls are excluded; i.e. -1 labels are passed through
    """

    if not xnull:
        lift = np.fromiter(((a == -1).any() for a in labels), dtype='i8')
        shape = np.asarray(shape, dtype='i8') + lift

    if not is_int64_overflow_possible(shape):
        # obs ids are deconstructable! take the fast route!
        out = decons_group_index(obs_ids, shape)
        return out if xnull or not lift.any() \
            else [x - y for x, y in zip(out, lift)]

    i = unique_label_indices(comp_ids)
    i8copy = lambda a: a.astype('i8', subok=False, copy=True)
    return [i8copy(lab[i]) for lab in labels] 
Example 5
Project: InSilicoSeq   Author: HadrienG   File: modeller.py    License: MIT License 6 votes vote down vote up
def divide_qualities_into_bins(qualities, n_bins=4):
    """Divides the raw quality scores in bins according to the mean phred
    quality of the sequence they come from

    Args:
        qualities (list): raw count of all the phred scores and mean sequence
            quality
        n_bins (int): number of bins to create (default: 4)

    Returns:
        list: a list of lists containing the binned quality scores
    """
    logger = logging.getLogger(__name__)
    logger.debug('Dividing qualities into mean clusters')
    bin_lists = [[] for _ in range(n_bins)]  # create list of `n_bins` list
    ranges = np.split(np.array(range(40)), n_bins)
    for quality in qualities:
        mean = int(quality[0][1])  # mean is at 1 and same regardless of b pos
        which_array = 0
        for array in ranges:
            if mean in array:
                read = np.fromiter((q[0] for q in quality), dtype=np.float)
                bin_lists[which_array].append(read)
            which_array += 1
    return bin_lists 
Example 6
Project: Pointnet_Pointnet2_pytorch   Author: yanx27   File: plyfile.py    License: MIT License 6 votes vote down vote up
def make2d(array, cols=None, dtype=None):
    '''
    Make a 2D array from an array of arrays.  The `cols' and `dtype'
    arguments can be omitted if the array is not empty.
    '''
    if (cols is None or dtype is None) and not len(array):
        raise RuntimeError("cols and dtype must be specified for empty "
                           "array")

    if cols is None:
        cols = len(array[0])

    if dtype is None:
        dtype = array[0].dtype

    return _np.fromiter(array, [('_', dtype, (cols,))],
                        count=len(array))['_'] 
Example 7
Project: vnpy_crypto   Author: birforce   File: test_array.py    License: MIT License 6 votes vote down vote up
def test_constructor_object_dtype(self):
        # GH 11856
        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object)
        assert arr.dtype == np.object
        assert np.isnan(arr.fill_value)

        arr = SparseArray(['A', 'A', np.nan, 'B'], dtype=np.object,
                          fill_value='A')
        assert arr.dtype == np.object
        assert arr.fill_value == 'A'

        # GH 17574
        data = [False, 0, 100.0, 0.0]
        arr = SparseArray(data, dtype=np.object, fill_value=False)
        assert arr.dtype == np.object
        assert arr.fill_value is False
        arr_expected = np.array(data, dtype=np.object)
        it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected))
        assert np.fromiter(it, dtype=np.bool).all() 
Example 8
Project: vnpy_crypto   Author: birforce   File: sorting.py    License: MIT License 6 votes vote down vote up
def decons_obs_group_ids(comp_ids, obs_ids, shape, labels, xnull):
    """
    reconstruct labels from observed group ids

    Parameters
    ----------
    xnull: boolean,
        if nulls are excluded; i.e. -1 labels are passed through
    """

    if not xnull:
        lift = np.fromiter(((a == -1).any() for a in labels), dtype='i8')
        shape = np.asarray(shape, dtype='i8') + lift

    if not is_int64_overflow_possible(shape):
        # obs ids are deconstructable! take the fast route!
        out = decons_group_index(obs_ids, shape)
        return out if xnull or not lift.any() \
            else [x - y for x, y in zip(out, lift)]

    i = unique_label_indices(comp_ids)
    i8copy = lambda a: a.astype('i8', subok=False, copy=True)
    return [i8copy(lab[i]) for lab in labels] 
Example 9
Project: BERT   Author: yyht   File: data_util_hdf5.py    License: Apache License 2.0 6 votes vote down vote up
def load_word2vec(word2vec_model_path,embed_size):
    """
    load pretrained word2vec in txt format
    :param word2vec_model_path:
    :return: word2vec_dict. word2vec_dict[word]=vector
    """
    #word2vec_object = codecs.open(word2vec_model_path,'r','utf-8') #open(word2vec_model_path,'r')
    #lines=word2vec_object.readlines()
    #word2vec_dict={}
    #for i,line in enumerate(lines):
    #    if i==0: continue
    #    string_list=line.strip().split(" ")
    #    word=string_list[0]
    #    vector=string_list[1:][0:embed_size]
    #    word2vec_dict[word]=vector
    ######################
    word2vec_dict = {}
    with open(word2vec_model_path, errors='ignore') as f:
        meta = f.readline()
        for line in f.readlines():
            items = line.split(' ')
            #if len(items[0]) > 1 and items[0] in vocab:
            word2vec_dict[items[0]] = np.fromiter(items[1:][0:embed_size], dtype=float)
    return word2vec_dict 
Example 10
Project: Computable   Author: ktraunmueller   File: util.py    License: MIT License 6 votes vote down vote up
def cartesian_product(X):
    '''
    Numpy version of itertools.product or pandas.compat.product.
    Sometimes faster (for large inputs)...

    Examples
    --------
    >>> cartesian_product([list('ABC'), [1, 2]])
    [array(['A', 'A', 'B', 'B', 'C', 'C'], dtype='|S1'),
 	array([1, 2, 1, 2, 1, 2])]

    '''

    lenX = np.fromiter((len(x) for x in X), dtype=int)
    cumprodX = np.cumproduct(lenX)

    a = np.roll(cumprodX, 1)
    a[0] = 1

    b = cumprodX[-1] / cumprodX

    return [np.tile(np.repeat(x, b[i]), 
                    np.product(a[i]))
               for i, x in enumerate(X)] 
Example 11
Project: pymesh   Author: taxpon   File: obj.py    License: MIT License 5 votes vote down vote up
def __load(fh):
        return numpy.fromiter(Obj.__read(fh), dtype=Obj.obj_dtype) 
Example 12
Project: pymesh   Author: taxpon   File: stl.py    License: MIT License 5 votes vote down vote up
def __load_ascii(fh, header):
        return numpy.fromiter(Stl.__ascii_reader(fh, header), dtype=Stl.stl_dtype) 
Example 13
Project: gnocchi   Author: gnocchixyz   File: utils.py    License: Apache License 2.0 5 votes vote down vote up
def to_timestamps(values):
    try:
        if len(values) == 0:
            return []
        if isinstance(values[0], (numpy.datetime64, datetime.datetime)):
            times = numpy.array(values)
        else:
            try:
                # Try to convert to float. If it works, then we consider
                # timestamps to be number of seconds since Epoch
                # e.g. 123456 or 129491.1293
                float(values[0])
            except ValueError:
                try:
                    # Try to parse the value as a string of ISO timestamp
                    # e.g. 2017-10-09T23:23:12.123
                    numpy.datetime64(values[0])
                except ValueError:
                    # Last chance: it can be relative timestamp, so convert
                    # to timedelta relative to now()
                    # e.g. "-10 seconds" or "5 minutes"
                    times = numpy.fromiter(
                        numpy.add(numpy.datetime64(utcnow()),
                                  [to_timespan(v, True) for v in values]),
                        dtype='datetime64[ns]', count=len(values))
                else:
                    times = numpy.array(values, dtype='datetime64[ns]')
            else:
                times = numpy.array(values, dtype='float') * 10e8
    except ValueError:
        raise ValueError("Unable to convert timestamps")

    times = times.astype('datetime64[ns]')

    if (times < unix_universal_start64).any():
        raise ValueError('Timestamp must be after Epoch')

    return times 
Example 14
Project: gnocchi   Author: gnocchixyz   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def _encode_measures(self, measures):
        return numpy.fromiter(measures,
                              dtype=TIMESERIES_ARRAY_DTYPE).tobytes() 
Example 15
Project: razzy-spinner   Author: rafasashi   File: hmm.py    License: GNU General Public License v3.0 5 votes vote down vote up
def _transitions_matrix(self):
        """ Return a matrix of transition log probabilities. """
        trans_iter = (self._transitions[sj].logprob(si)
                      for sj in self._states
                      for si in self._states)

        transitions_logprob = np.fromiter(trans_iter, dtype=np.float64)
        N = len(self._states)
        return transitions_logprob.reshape((N, N)).T 
Example 16
Project: razzy-spinner   Author: rafasashi   File: hmm.py    License: GNU General Public License v3.0 5 votes vote down vote up
def _outputs_vector(self, symbol):
        """
        Return a vector with log probabilities of emitting a symbol
        when entering states.
        """
        out_iter = (self._output_logprob(sj, symbol) for sj in self._states)
        return np.fromiter(out_iter, dtype=np.float64) 
Example 17
Project: radiometric_normalization   Author: planetlabs   File: pca_filter.py    License: Apache License 2.0 5 votes vote down vote up
def _numpy_array_from_2arrays(array1, array2, dtype=numpy.uint16):
    ''' Efficiently combine two 1-D arrays into a single 2-D array.

    Avoids large memory usage by creating the array using
    ``numpy.fromiter`` and then reshaping a view of the resulting
    record array.  This does the equivalent of:

        numpy.array(zip(array1, array2))

    but avoids holding a potentially large number of tuples in memory.

    >>> a = numpy.array([1, 2, 3])
    >>> b = numpy.array([4, 5, 6])
    >>> X = _numpy_array_from_2arrays(a, b)
    >>> X
    array([[1, 4],
           [2, 5],
           [3, 6]], dtype=uint16)

    :param array array1: A 1-D numpy array.
    :param array array2: A second 1-D numpy array.
    :param data-type dtype: Data type for array elements
        (must be same for both arrays)

    :returns: A 2D numpy array combining the two input arrays
    '''
    array_dtype = [('x', dtype), ('y', dtype)]

    return numpy.fromiter(itertools.izip(array1, array2), dtype=array_dtype) \
                .view(dtype=dtype) \
                .reshape((-1, 2)) 
Example 18
Project: mabwiser   Author: fidelity   File: thompson.py    License: Apache License 2.0 5 votes vote down vote up
def _get_binary_rewards(self, decisions: np.ndarray, rewards: np.ndarray):

        # If a binarizer function is given and binarization has not taken place already in a neighborhood policy
        if self.binarizer and not self.is_contextual_binarized:
            return np.fromiter((self.binarizer(decisions[index], value)  # convert every decision-reward pair to binary
                                for index, value in enumerate(rewards)), rewards.dtype)
        else:
            return rewards 
Example 19
Project: recruit   Author: Frank-qlu   File: test_numeric.py    License: Apache License 2.0 5 votes vote down vote up
def test_types(self):
        ai32 = np.fromiter(self.makegen(), np.int32)
        ai64 = np.fromiter(self.makegen(), np.int64)
        af = np.fromiter(self.makegen(), float)
        assert_(ai32.dtype == np.dtype(np.int32))
        assert_(ai64.dtype == np.dtype(np.int64))
        assert_(af.dtype == np.dtype(float)) 
Example 20
Project: recruit   Author: Frank-qlu   File: test_numeric.py    License: Apache License 2.0 5 votes vote down vote up
def test_lengths(self):
        expected = np.array(list(self.makegen()))
        a = np.fromiter(self.makegen(), int)
        a20 = np.fromiter(self.makegen(), int, 20)
        assert_(len(a) == len(expected))
        assert_(len(a20) == 20)
        assert_raises(ValueError, np.fromiter,
                          self.makegen(), int, len(expected) + 10) 
Example 21
Project: recruit   Author: Frank-qlu   File: test_numeric.py    License: Apache License 2.0 5 votes vote down vote up
def test_values(self):
        expected = np.array(list(self.makegen()))
        a = np.fromiter(self.makegen(), int)
        a20 = np.fromiter(self.makegen(), int, 20)
        assert_(np.alltrue(a == expected, axis=0))
        assert_(np.alltrue(a20 == expected[:20], axis=0)) 
Example 22
Project: recruit   Author: Frank-qlu   File: test_numeric.py    License: Apache License 2.0 5 votes vote down vote up
def test_2592(self):
        # Test iteration exceptions are correctly raised.
        count, eindex = 10, 5
        assert_raises(NIterError, np.fromiter,
                          self.load_data(count, eindex), dtype=int, count=count) 
Example 23
Project: recruit   Author: Frank-qlu   File: test_numeric.py    License: Apache License 2.0 5 votes vote down vote up
def test_2592_edge(self):
        # Test iter. exceptions, edge case (exception at end of iterator).
        count = 10
        eindex = count-1
        assert_raises(NIterError, np.fromiter,
                          self.load_data(count, eindex), dtype=int, count=count) 
Example 24
Project: recruit   Author: Frank-qlu   File: test_regression.py    License: Apache License 2.0 5 votes vote down vote up
def test_mem_fromiter_invalid_dtype_string(self):
        x = [1, 2, 3]
        assert_raises(ValueError,
                              np.fromiter, [xi for xi in x], dtype='S') 
Example 25
Project: recruit   Author: Frank-qlu   File: test_regression.py    License: Apache License 2.0 5 votes vote down vote up
def test_fromiter_bytes(self):
        # Ticket #1058
        a = np.fromiter(list(range(10)), dtype='b')
        b = np.fromiter(list(range(10)), dtype='B')
        assert_(np.alltrue(a == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])))
        assert_(np.alltrue(b == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))) 
Example 26
Project: recruit   Author: Frank-qlu   File: test_regression.py    License: Apache License 2.0 5 votes vote down vote up
def test_fromiter_comparison(self):
        a = np.fromiter(list(range(10)), dtype='b')
        b = np.fromiter(list(range(10)), dtype='B')
        assert_(np.alltrue(a == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])))
        assert_(np.alltrue(b == np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))) 
Example 27
Project: recruit   Author: Frank-qlu   File: test_regression.py    License: Apache License 2.0 5 votes vote down vote up
def test_duplicate_field_names_assign(self):
        ra = np.fromiter(((i*3, i*2) for i in range(10)), dtype='i8,f8')
        ra.dtype.names = ('f1', 'f2')
        repr(ra)  # should not cause a segmentation fault
        assert_raises(ValueError, setattr, ra.dtype, 'names', ('f1', 'f1')) 
Example 28
Project: recruit   Author: Frank-qlu   File: stata.py    License: Apache License 2.0 5 votes vote down vote up
def _prepare_data(self):
        data = self.data
        typlist = self.typlist
        convert_dates = self._convert_dates
        # 1. Convert dates
        if self._convert_dates is not None:
            for i, col in enumerate(data):
                if i in convert_dates:
                    data[col] = _datetime_to_stata_elapsed_vec(data[col],
                                                               self.fmtlist[i])
        # 2. Convert strls
        data = self._convert_strls(data)

        # 3. Convert bad string data to '' and pad to correct length
        dtypes = []
        data_cols = []
        has_strings = False
        native_byteorder = self._byteorder == _set_endianness(sys.byteorder)
        for i, col in enumerate(data):
            typ = typlist[i]
            if typ <= self._max_string_length:
                has_strings = True
                data[col] = data[col].fillna('').apply(_pad_bytes, args=(typ,))
                stype = 'S{type}'.format(type=typ)
                dtypes.append(('c' + str(i), stype))
                string = data[col].str.encode(self._encoding)
                data_cols.append(string.values.astype(stype))
            else:
                values = data[col].values
                dtype = data[col].dtype
                if not native_byteorder:
                    dtype = dtype.newbyteorder(self._byteorder)
                dtypes.append(('c' + str(i), dtype))
                data_cols.append(values)
        dtypes = np.dtype(dtypes)

        if has_strings or not native_byteorder:
            self.data = np.fromiter(zip(*data_cols), dtype=dtypes)
        else:
            self.data = data.to_records(index=False) 
Example 29
Project: pytorch_geometric   Author: rusty1s   File: gdc.py    License: MIT License 5 votes vote down vote up
def __neighbors_to_graph__(self, neighbors, neighbor_weights,
                               normalization='row', device='cpu'):
        r"""Combine a list of neighbors and neighbor weights to create a sparse
        graph.

        Args:
            neighbors (List[List[int]]): List of neighbors for each node.
            neighbor_weights (List[List[float]]): List of weights for the
                neighbors of each node.
            normalization (str): Normalization of resulting matrix
                (options: :obj:`"row"`, :obj:`"col"`). (default: :obj:`"row"`)
            device (torch.device): Device to create output tensors on.
                (default: :obj:`"cpu"`)

        :rtype: (:class:`LongTensor`, :class:`Tensor`)
        """
        edge_weight = torch.Tensor(np.concatenate(neighbor_weights)).to(device)
        i = np.repeat(np.arange(len(neighbors)),
                      np.fromiter(map(len, neighbors), dtype=np.int))
        j = np.concatenate(neighbors)
        if normalization == 'col':
            edge_index = torch.Tensor(np.vstack([j, i])).to(device)
            N = len(neighbors)
            edge_index, edge_weight = coalesce(edge_index, edge_weight, N, N)
        elif normalization == 'row':
            edge_index = torch.Tensor(np.vstack([i, j])).to(device)
        else:
            raise ValueError(
                f"PPR matrix normalization {normalization} unknown.")
        return edge_index, edge_weight 
Example 30
Project: segpy   Author: sixty-north   File: extract.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def extract_trace_headers(reader, fields, trace_indexes=None):
    """Extract trace header fields from the specified trace headers as separate arrays.

    Args:
        reader: A SegYReader

        fields: A an iterable series where each item is either the name of a field as a string
            or an object such as a NamedField with a 'name' attribute which in turn is the name
            of a field as a string, such as a NamedField.

        trace_indexes: An optional iterable series of trace_indexes. If not provided or None,
            the headers for all trace indexes will be returned.

    Returns:
        A namedtuple with attributes which are one-dimensionsal Numpy arrays.
    """
    if trace_indexes is None:
        trace_indexes = reader.trace_indexes()

    field_names = [_extract_field_name(field) for field in fields]

    class SubFormat(metaclass=SubFormatMeta,
                    parent_format=reader.trace_header_format_class,
                    parent_field_names=field_names):
        pass

    sub_header_packer = make_header_packer(SubFormat, reader.endian)
    trace_header_arrays_cls = namedtuple('trace_header_arrays_cls', field_names)

    trace_headers = [reader.trace_header(trace_index, sub_header_packer)
                     for trace_index in trace_indexes]

    trace_header_arrays = trace_header_arrays_cls(
        *(np.fromiter((getattr(trace_header, field_name) for trace_header in trace_headers),
                      dtype=make_dtype(getattr(SubFormat, field_name).value_type.SEG_Y_TYPE),
                      count=len(trace_headers))
          for field_name in field_names)
    )

    return trace_header_arrays