Python numpy.ndarrays() Examples

The following are 30 code examples of numpy.ndarrays(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module numpy , or try the search function .
Example #1
Source File: quantization.py    From SNIPER-mxnet with Apache License 2.0 6 votes vote down vote up
def _get_optimal_thresholds(nd_dict, num_bins=8001, num_quantized_bins=255, logger=None):
    """Given a ndarray dict, find the optimal threshold for quantizing each value of the key."""
    if stats is None:
        raise ImportError('scipy.stats is required for running entropy mode of calculating'
                          ' the optimal thresholds for quantizing FP32 ndarrays into int8.'
                          ' Please check if the scipy python bindings are installed.')
    assert isinstance(nd_dict, dict)
    if logger is not None:
        logger.info('Calculating optimal thresholds for quantization using KL divergence'
                    ' with num_bins=%d and num_quantized_bins=%d' % (num_bins, num_quantized_bins))
    th_dict = {}
    # copy nd_dict keys since the keys() only returns a view in python3
    layer_names = list(nd_dict.keys())
    for name in layer_names:
        assert name in nd_dict
        min_val, max_val, min_divergence, opt_th =\
            _get_optimal_threshold(nd_dict[name], num_bins=num_bins,
                                   num_quantized_bins=num_quantized_bins)
        del nd_dict[name]  # release the memory of ndarray
        th_dict[name] = (-opt_th, opt_th)
        if logger is not None:
            logger.info('layer=%s, min_val=%f, max_val=%f, min_divergence=%f, optimal_threshold=%f'
                        % (name, min_val, max_val, min_divergence, opt_th))
    return th_dict 
Example #2
Source File: tools.py    From vnpy_crypto with MIT License 6 votes vote down vote up
def nan_dot(A, B):
    """
    Returns np.dot(left_matrix, right_matrix) with the convention that
    nan * 0 = 0 and nan * x = nan if x != 0.

    Parameters
    ----------
    A, B : np.ndarrays
    """
    # Find out who should be nan due to nan * nonzero
    should_be_nan_1 = np.dot(np.isnan(A), (B != 0))
    should_be_nan_2 = np.dot((A != 0), np.isnan(B))
    should_be_nan = should_be_nan_1 + should_be_nan_2

    # Multiply after setting all nan to 0
    # This is what happens if there were no nan * nonzero conflicts
    C = np.dot(np.nan_to_num(A), np.nan_to_num(B))

    C[should_be_nan] = np.nan

    return C 
Example #3
Source File: test_case.py    From graphics with Apache License 2.0 6 votes vote down vote up
def _max_error(arrays1, arrays2):
  """Computes maximum elementwise gap between two lists of ndarrays.

  Computes the maximum elementwise gap between two lists with the same length,
  of arrays with the same shape.

  Args:
    arrays1: a lists of np.ndarrays.
    arrays2: a lists of np.ndarrays of the same shape as arrays1.

  Returns:
    The maximum elementwise absolute difference between the two lists of arrays.
  """
  error = 0
  for array1, array2 in zip(arrays1, arrays2):
    if array1.size or array2.size:  # Handle zero size ndarrays correctly
      error = np.maximum(error, np.fabs(array1 - array2).max())
  return error 
Example #4
Source File: engine.py    From moonlight with Apache License 2.0 6 votes vote down vote up
def _nested_ndarrays_to_tensors(data):
  """Converts possibly nested lists of np.ndarrays and Tensors to Tensors.

  This is necessary in case some data in the Structure is already computed. We
  just pass everything to tf.Session.run, and some data may be a tf.constant
  which is just spit back out.

  Args:
    data: An np.ndarray, Tensor, or list recursively containing the same types.

  Returns:
    data with all np.ndarrays converted to Tensors.

  Raises:
    ValueError: If unexpected data was given.
  """
  if isinstance(data, list):
    return [_nested_ndarrays_to_tensors(element) for element in data]
  elif isinstance(data, np.ndarray):
    return tf.constant(data)
  elif isinstance(data, tf.Tensor):
    return data
  else:
    raise ValueError('Unexpected data: %s' % data) 
Example #5
Source File: recurrentstate.py    From theanolm with Apache License 2.0 6 votes vote down vote up
def __init__(self, sizes, num_sequences=1, state_variables=None):
        """Constructs a list of recurrent layer states and initializes them.

        This will create state vectors for each layer for each of the sequences
        that will be processed in parallel. Unless ``state_variables`` is given,
        the vectors will be initialized to zeros.

        :type sizes: list of ints
        :param sizes: size of each recurrent layer state

        :type num_sequences: int
        :param num_sequences: number of sequences to be processed in parallel

        :type state_variables: list of numpy.ndarrays
        :param state_variables: if set to other than ``None``, sets the initial
                                recurrent layer states to this instead of zeros
        """

        self.sizes = sizes
        self.num_sequences = num_sequences
        if state_variables is None:
            self.reset()
        else:
            self.set(state_variables) 
Example #6
Source File: recurrentstate.py    From theanolm with Apache License 2.0 6 votes vote down vote up
def set(self, state_variables):
        """Sets the state vector of every recurrent layer.

        :type state_variables: list of numpy.ndarrays
        :param state_variables: a matrix for each recurrent layer that contains
                                the state vector for each sequence at one time
                                step
        """

        if len(state_variables) != len(self.sizes):
            raise ValueError("Recurrent state should contain as many arrays "
                             "as there are recurrent layers.")
        for state_variable, size in zip(state_variables, self.sizes):
            if state_variable.shape[0] != 1:
                raise ValueError("Recurrent state should contain only one time "
                                 "step.")
            if state_variable.shape[1] != self.num_sequences:
                raise ValueError("Recurrent state contains incorrect number of "
                                 "sequences.")
            if state_variable.shape[2] != size:
                raise ValueError("Recurrent state contains a layer with "
                                 "incorrect size.")
        self._state_variables = state_variables 
Example #7
Source File: recurrentstate.py    From theanolm with Apache License 2.0 6 votes vote down vote up
def get(self, index=None):
        """Returns the state matrix of a given layer, or a list of the matrices
        of all layers.

        :type index: int
        :param index: index of a recurrent layer in the state object; if
                            set to other than ``None``, returns only the matrix
                            for the corresponding layer

        :rtype: numpy.ndarray or list of numpy.ndarrays
        :returns: a matrix for each recurrent layer that contains the state
                  vector for each sequence at one time step
        """

        if index is not None:
            return self._state_variables[index]
        else:
            return self._state_variables 
Example #8
Source File: tools.py    From Splunking-Crime with GNU Affero General Public License v3.0 6 votes vote down vote up
def nan_dot(A, B):
    """
    Returns np.dot(left_matrix, right_matrix) with the convention that
    nan * 0 = 0 and nan * x = nan if x != 0.

    Parameters
    ----------
    A, B : np.ndarrays
    """
    # Find out who should be nan due to nan * nonzero
    should_be_nan_1 = np.dot(np.isnan(A), (B != 0))
    should_be_nan_2 = np.dot((A != 0), np.isnan(B))
    should_be_nan = should_be_nan_1 + should_be_nan_2

    # Multiply after setting all nan to 0
    # This is what happens if there were no nan * nonzero conflicts
    C = np.dot(np.nan_to_num(A), np.nan_to_num(B))

    C[should_be_nan] = np.nan

    return C 
Example #9
Source File: dataframe.py    From skoot with MIT License 6 votes vote down vote up
def safe_vstack(a, b):
    """Stack two arrays on top of one another.

    Safely handle vertical stacking of arrays. This works for
    either np.ndarrays or pd.DataFrames. The types of both inputs must
    match!

    Parameters
    ----------
    a : array-like, shape=(n_samples, n_features)
        The array that will be stacked on the top vertically.

    b : array-like, shape=(n_samples, n_features)
        The array that will be stacked below the other vertically.
    """
    # we can only pd.concat if they BOTH are DataFrames
    if all(isinstance(x, pd.DataFrame) for x in (a, b)):
        return pd.concat([a, b], axis=0)

    # otherwise, at least one of them is a numpy array (we think)
    return np.vstack([a, b]) 
Example #10
Source File: grid.py    From orbkit with GNU Lesser General Public License v3.0 6 votes vote down vote up
def mv2g(**kwargs):
  '''Converts all `numpy.ndarrays` given as the keyword arguments 
  (`**kwargs`) from a vector grid of `shape=(..., Nx*Ny*Nz, ...,)` to a regular 
  grid of `shape=(..., Nx, Ny, Nz, ...,)`, and, if more than one `**kwargs` is 
  given, returns it as a dictionary.
  
  Hint: The global values for the grid dimensionality, i.e., :mod:`grid.N_`,
  are used for reshaping.
  '''
  import itertools
  return_val = {}
  for i,j in kwargs.items():
    j = numpy.asarray(j,dtype=float)
    shape = numpy.shape(j)
    where = numpy.argwhere(shape==numpy.product(N_))[0,0]
    return_val[i] = numpy.zeros(shape[:where]+tuple(N_)+shape[where+1:])
    for key in itertools.product(*[range(k) for k in (shape[:where] + shape[where+1:])]):
      obj = [slice(k,k+1) for k in key]
      for r in range(3):
        obj.insert(where,slice(None,None))
      return_val[i][obj] = matrix_vector2grid(j[obj[:where]+obj[where+2:]].reshape((-1,)), 
                                          **dict(zip(['Nx','Ny','Nz'],N_)))

  return list(return_val.values())[0] if len(return_val.values()) == 1 else return_val 
Example #11
Source File: ranking.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def get_newsnr_sgveto(trigs):
    """
    Calculate newsnr re-weigthed by the sine-gaussian veto

    Parameters
    ----------
    trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
        Dictionary-like object holding single detector trigger information.
        'chisq_dof', 'snr', 'sg_chisq' and 'chisq' are required keys

    Returns
    -------
    numpy.ndarray
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr_sg = newsnr_sgveto(trigs['snr'][:],
                            trigs['chisq'][:] / dof,
                            trigs['sg_chisq'][:])
    return numpy.array(nsnr_sg, ndmin=1, dtype=numpy.float32) 
Example #12
Source File: ranking.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def get_newsnr_sgveto_psdvar(trigs):
    """
    Calculate snr re-weighted by Allen chisq, sine-gaussian veto and
    psd variation statistic

    Parameters
    ----------
    trigs: dict of numpy.ndarrays
        Dictionary holding single detector trigger information.
    'chisq_dof', 'snr', 'chisq' and 'psd_var_val' are required keys

    Returns
    -------
     numpy.ndarray
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr_sg_psd = \
                 newsnr_sgveto_psdvar(trigs['snr'][:], trigs['chisq'][:] / dof,
                                      trigs['sg_chisq'][:],
                                      trigs['psd_var_val'][:])
    return numpy.array(nsnr_sg_psd, ndmin=1, dtype=numpy.float32) 
Example #13
Source File: ranking.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def get_newsnr_sgveto_psdvar_scaled(trigs):
    """
    Calculate newsnr re-weighted by the sine-gaussian veto and scaled
    psd variation statistic

    Parameters
    ----------
    trigs: dict of numpy.ndarrays
        Dictionary holding single detector trigger information.
    'chisq_dof', 'snr', 'chisq' and 'psd_var_val' are required keys

    Returns
    -------
     numpy.ndarray
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr_sg_psdscale = \
                 newsnr_sgveto_psdvar_scaled(
                     trigs['snr'][:], trigs['chisq'][:] / dof,
                     trigs['sg_chisq'][:],
                     trigs['psd_var_val'][:])
    return numpy.array(nsnr_sg_psdscale, ndmin=1, dtype=numpy.float32) 
Example #14
Source File: ranking.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def get_newsnr_sgveto_psdvar_scaled_threshold(trigs):
    """
    Calculate newsnr re-weighted by the sine-gaussian veto and scaled
    psd variation statistic. A further threshold is applied to the
    reduced chisq.

    Parameters
    ----------
    trigs: dict of numpy.ndarrays
        Dictionary holding single detector trigger information.
    'chisq_dof', 'snr', 'chisq' and 'psd_var_val' are required keys

    Returns
    -------
     numpy.ndarray
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr_sg_psdt = \
                 newsnr_sgveto_psdvar_scaled_threshold(
                     trigs['snr'][:], trigs['chisq'][:] / dof,
                     trigs['sg_chisq'][:],
                     trigs['psd_var_val'][:])
    return numpy.array(nsnr_sg_psdt, ndmin=1, dtype=numpy.float32) 
Example #15
Source File: stat.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def single(self, trigs):
        """Calculate the single detector statistic.

        Parameters
        ----------
        trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
            Dictionary-like object holding single detector trigger information.

        Returns
        -------
        newsnr: numpy.ndarray
            Array of single detector values
        """
        newsnr = ranking.get_newsnr(trigs)
        rchisq = trigs['chisq'][:] / (2. * trigs['chisq_dof'][:] - 2.)
        newsnr[numpy.logical_and(newsnr < 10, rchisq > 2)] = -1
        return newsnr 
Example #16
Source File: stat.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def single(self, trigs):
        """Calculate the single detector statistic & assemble other parameters

        Parameters
        ----------
        trigs: dict of numpy.ndarrays, h5py group or similar dict-like object
            Object holding single detector trigger information. 'snr', 'chisq',
        'chisq_dof', 'coa_phase', 'end_time', and 'sigmasq' are required keys.

        Returns
        -------
        numpy.ndarray
            Array of single detector parameter values
        """
        sngl_stat = self.get_newsnr(trigs)
        singles = numpy.zeros(len(sngl_stat), dtype=self.single_dtype)
        singles['snglstat'] = sngl_stat
        singles['coa_phase'] = trigs['coa_phase'][:]
        singles['end_time'] = trigs['end_time'][:]
        singles['sigmasq'] = trigs['sigmasq'][:]
        singles['snr'] = trigs['snr'][:]
        return numpy.array(singles, ndmin=1) 
Example #17
Source File: stat.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def single(self, trigs):
        """Calculate the single detector statistic & assemble other parameters

        Parameters
        ----------
        trigs: dict of numpy.ndarrays, h5py group or similar dict-like object
            Object holding single detector trigger information. 'snr', 'chisq',
        'chisq_dof', 'coa_phase', 'end_time', and 'sigmasq' are required keys.

        Returns
        -------
        numpy.ndarray
            Array of single detector parameter values
        """
        sngl_stat = self.get_newsnr(trigs)
        singles = numpy.zeros(len(sngl_stat), dtype=self.single_dtype)
        singles['snglstat'] = sngl_stat
        singles['coa_phase'] = trigs['coa_phase'][:]
        singles['end_time'] = trigs['end_time'][:]
        singles['sigmasq'] = trigs['sigmasq'][:]
        singles['snr'] = trigs['snr'][:]
        return numpy.array(singles, ndmin=1) 
Example #18
Source File: ranking.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def get_newsnr(trigs):
    """
    Calculate newsnr ('reweighted SNR') for a trigs/dictionary object

    Parameters
    ----------
    trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
        Dictionary-like object holding single detector trigger information.
        'chisq_dof', 'snr', and 'chisq' are required keys

    Returns
    -------
    numpy.ndarray
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr = newsnr(trigs['snr'][:], trigs['chisq'][:] / dof)
    return numpy.array(nsnr, ndmin=1, dtype=numpy.float32) 
Example #19
Source File: stat.py    From pycbc with GNU General Public License v3.0 6 votes vote down vote up
def single(self, trigs):
        """Calculate the single detector statistic.

        Parameters
        ----------
        trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
            Dictionary-like object holding single detector trigger information.
            'snr', 'cont_chisq', 'cont_chisq_dof', 'chisq_dof' and 'chisq'
            are required keys for this statistic.

        Returns
        -------
        stat: numpy.ndarray
            The array of single detector values
        """
        chisq_newsnr = ranking.get_newsnr(trigs)
        rautochisq = trigs['cont_chisq'][:] / trigs['cont_chisq_dof'][:]
        autochisq_newsnr = ranking.newsnr(trigs['snr'][:], rautochisq)
        return numpy.array(numpy.minimum(chisq_newsnr, autochisq_newsnr,
                           dtype=numpy.float32), ndmin=1, copy=False) 
Example #20
Source File: pytorch_ext.py    From L3C-PyTorch with GNU General Public License v3.0 6 votes vote down vote up
def histogram(t, L):
    """
    A: If t is a list of tensors/np.ndarrays, B is executed for all, yielding len(ts) histograms, which are summed
    per bin
    B: convert t to numpy, count bins.
    :param t: tensor or list of tensor, each expected to be in [0, L)
    :param L: number of symbols
    :return: length-L array, containing at l the number of values mapping to to symbol l
    """
    if isinstance(t, list):
        ts = t
        histograms = np.stack((histogram(t, L) for t in ts), axis=0)  # get array (len(ts) x L)
        return np.sum(histograms, 0)
    assert 0 <= t.min() and t.max() < L, (t.min(), t.max())
    a = tensor_to_np(t)
    counts, _ = np.histogram(a, np.arange(L+1))  # +1 because np.histogram takes bin edges, including rightmost edge
    return counts


# Gradients -------------------------------------------------------------------- 
Example #21
Source File: util.py    From sigver with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def process_dataset(dataset: IterableDataset,
                    save_path: str,
                    img_size: Tuple[int, int],
                    subset: slice = slice(None)):
    """ Processes a dataset (normalizing the images) and saves the result as a
        numpy npz file (collection of np.ndarrays).

    Parameters
    ----------
    dataset : IterableDataset
        The dataset, that knows where the signature files are located
    save_path : str
        The name of the file to save the numpy arrays
    img_size : tuple (H x W)
        The final size of the images
    subset : slice
        Which users to consider. e.g. slice(None) to consider all users, or slice(first, last)


    Returns
    -------
    None

    """
    preprocess_fn = functools.partial(preprocess_signature,
                                      canvas_size=dataset.maxsize,
                                      img_size=img_size,
                                      input_size=img_size)

    processed = process_dataset_images(dataset, preprocess_fn, img_size, subset)
    x, y, yforg, user_mapping, used_files = processed

    np.savez(save_path,
             x=x,
             y=y,
             yforg=yforg,
             user_mapping=user_mapping,
             filenames=used_files) 
Example #22
Source File: stat.py    From pycbc with GNU General Public License v3.0 5 votes vote down vote up
def single(self, trigs):
        """Calculate the single detector statistic, here equal to newsnr

        Parameters
        ----------
        trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
            Dictionary-like object holding single detector trigger information.

        Returns
        -------
        numpy.ndarray
            The array of single detector values
        """
        return ranking.get_newsnr(trigs) 
Example #23
Source File: history.py    From pyPESTO with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self,
                 file: str,
                 x_names: Sequence[str] = None,
                 options: Union[HistoryOptions, Dict] = None,
                 load_from_file: bool = False):
        super().__init__(options=options)
        self.x_names = x_names
        self._trace: Union[pd.DataFrame, None] = None
        self.file = os.path.abspath(file)

        # create trace file dirs
        if self.file is not None:
            dirname = os.path.dirname(self.file)
            os.makedirs(dirname, exist_ok=True)

        if load_from_file:
            trace = pd.read_csv(self.file, header=[0, 1], index_col=0)
            # replace 'nan' in cols with np.NAN
            cols = pd.DataFrame(trace.columns.to_list())
            cols[cols == 'nan'] = np.NaN
            trace.columns = pd.MultiIndex.from_tuples(
                cols.to_records(index=False).tolist()
            )
            for col in trace.columns:
                # transform strings to np.ndarrays
                trace[col] = trace[col].apply(string2ndarray)

            self._trace = trace
            self.x_names = trace[X].columns
            self._update_counts_from_trace() 
Example #24
Source File: pre_post_process.py    From pyPESTO with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def as_ndarrays(
            result: Dict
    ) -> Dict:
        """
        Convert all array_like objects to np.ndarrays. This has the advantage
        of a uniform output datatype which offers various methods to assess
        the data.
        """
        keys = [GRAD, HESS, RES, SRES]
        for key in keys:
            if key in result:
                value = result[key]
                if value is not None:
                    result[key] = np.array(value)
        return result 
Example #25
Source File: loom_view.py    From loompy with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __getitem__(self, slice_: Union[str, Tuple[Union[int, np.ndarray, slice], Union[int, np.ndarray, slice]]]) -> np.ndarray:
		"""
		Get a slice of the main matrix.
		Args:
			slice:		A 2D slice object (see http://docs.h5py.org/en/latest/high/dataset.html) or np.ndarrays or ints
		Returns:
			A numpy matrix
		"""
		if type(slice_) is str:
			return self.layers[slice_]
		else:
			return self.layers[""][slice_] 
Example #26
Source File: TFGP.py    From handful-of-trials with MIT License 5 votes vote down vote up
def predict(self, inputs, *args, **kwargs):
        """Returns the predictions of this model on inputs.

        Arguments:
            inputs: (np.ndarray) The inputs on which predictions will be returned.
            ign_var: (bool) If True, only returns the mean prediction

        Returns: (np.ndarrays) The mean and variance of the model on the new points.
        """
        if self.model is None:
            raise RuntimeError("Cannot make predictions without initial batch of data.")

        with self.sess.as_default():
            mean, var = self.model.predict_y(inputs)
            return mean, var 
Example #27
Source File: space_utils.py    From rlgraph with Apache License 2.0 5 votes vote down vote up
def horizontalize_space_sample(space, sample, batch_size=1):
    # For Dicts, we have to treat each key as an array with batch-rank at index 0.
    # The dict is then translated into a list of dicts where each dict contains the original data
    # but without the batch-rank.
    # E.g. {'A': array([0, 1]), 'B': array([2, 3])} -> [{'A': 0, 'B': 2}, {'A': 1, 'B': 3}]
    if isinstance(space, Dict):
        some_key = next(iter(sample))
        assert isinstance(sample, dict) and isinstance(sample[some_key], np.ndarray), \
            "ERROR: Cannot flip Dict batch with dict keys if returned value is not a dict OR " \
            "values of returned value are not np.ndarrays!"
        # TODO: What if actions come as nested dicts (more than one level deep)?
        # TODO: Use DataOpDict/Tuple's new `map` method.
        if hasattr(sample[some_key], "__len__"):
            result = [{key: value[i] for key, value in sample.items()} for i in range(len(sample[some_key]))]
        else:
            # Action was not array type.
            result = [{key: value for key, value in sample.items()}]
    # Tuple:
    # E.g. Tuple(array([0, 1]), array([2, 3])) -> [(0, 2), (1, 3)]
    elif isinstance(space, Tuple):
        assert isinstance(sample, tuple) and isinstance(sample[0], np.ndarray), \
            "ERROR: Cannot flip tuple batch if returned value is not a tuple OR " \
            "values of returned value are not np.ndarrays!"
        # TODO: Use DataOpDict/Tuple's new `map` method.
        result = [tuple(value[i] for _, value in enumerate(sample)) for i in range(len(sample[0]))]
    # No container batch-flipping necessary.
    else:
        result = sample
        if batch_size == 1 and result.shape == ():
            result = [result]

    return result 
Example #28
Source File: _act.py    From skutil with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _as_numpy(*args):
    """Given an iterable (a 1d list, np.ndarray, pd.Series, 
    pd.DataFrame or H2OFrame), convert it into a 1d np.ndarray
    for further processing.

    Returns
    -------
    arrs : list
        Returns a list (of 1d np.ndarrays) of length==len(args)
    """
    def _single_as_numpy(x):
        if not isinstance(x, np.ndarray):
            # if an H2OFrame, just return the first col
            if isinstance(x, H2OFrame):
                # same as ..h2o.util.h2o_col_to_numpy, but
                # that causes circular dependency in imports.
                if not x.shape[1] == 1:
                    raise ValueError('must be 1d column')
                _1d = x[x.columns[0]].as_data_frame(use_pandas=True)
                return _1d[_1d.columns[0]].values
            elif is_iterable(x):
                return np.asarray(x)
            else:
                raise TypeError('cannot create numpy array out of type=%s' % type(x))
        else:
            return np.copy(x)

    arrs = [_single_as_numpy(i) for i in args]
    if len(arrs) == 1:
        arrs = arrs[0]

    return arrs 
Example #29
Source File: coinc.py    From pycbc with GNU General Public License v3.0 5 votes vote down vote up
def backout_last(self, updated_singles, num_coincs):
        """Remove the recently added singles and coincs

        Parameters
        ----------
        updated_singles: dict of numpy.ndarrays
            Array of indices that have been just updated in the internal
            buffers of single detector triggers.
        num_coincs: int
            The number of coincs that were just added to the internal buffer
            of coincident triggers
        """
        for ifo in updated_singles:
            self.singles[ifo].discard_last(updated_singles[ifo])
        self.coincs.remove(num_coincs) 
Example #30
Source File: process_dataset.py    From sigver with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def process_dataset(dataset: IterableDataset,
                    save_path: str,
                    img_size: Tuple[int, int],
                    subset: Optional[slice] = None):
    """ Processes a dataset (normalizing the images) and saves the result as a
        numpy npz file (collection of np.ndarrays).

    Parameters
    ----------
    dataset : IterableDataset
        The dataset, that knows where the signature files are located
    save_path : str
        The name of the file to save the numpy arrays
    img_size : tuple (H x W)
        The final size of the images
    subset : slice
        Which users to consider. e.g. slice(None) to consider all users, or slice(first, last)


    Returns
    -------
    None

    """
    preprocess_fn = functools.partial(preprocess_signature,
                                      canvas_size=dataset.maxsize,
                                      img_size=img_size,
                                      input_size=img_size) # Don't crop it now

    if subset is None:
        subset = slice(None) # Use all
    processed = process_dataset_images(dataset, preprocess_fn, img_size, subset)
    x, y, yforg, user_mapping, used_files = processed

    np.savez(save_path,
             x=x,
             y=y,
             yforg=yforg,
             user_mapping=user_mapping,
             filenames=used_files)