Python numpy.argsort() Examples

The following are code examples for showing how to use numpy.argsort(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: PyTorch-Sentiment-Analysis-deployed-with-Flask   Author: oliverproud   File: model.py    MIT License 7 votes vote down vote up
def predict_sentiment(sentence, model=model,min_len = 5):

    model.eval()

    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]

    '''
    If the length of the sentence is shorter than the length of the largest filter
    then the sentence must be padded to the length of the largest filter
    '''
    if len(tokenized) < min_len:
        tokenized += ['<pad>'] * (min_len - len(tokenized))
    indexed = [TEXT.stoi[t] for t in tokenized]
    tensor = torch.LongTensor(indexed)
    tensor = tensor.unsqueeze(1)
    prediction = torch.sigmoid(model(tensor))

    probs = [{'name': index, 'prob': prediction.item()} for index in np.argsort(prediction.item())]

    return (sentence, probs) 
Example 2
Project: building-boundary   Author: Geodan   File: segment.py    MIT License 7 votes vote down vote up
def PCA(points):
    """
    Does a Principle Component Analysis (PCA) for a set of 3D
    points (the structure tensor) by computing the eigenvalues
    and eigenvectors of the covariance matrix of a point cloud.

    Parameters
    ----------
    points : (Mx3) array
        X, Y and Z coordinates of points.

    Returns
    -------
    eigenvalues : (1x3) array
        The eigenvalues corrisponding to the eigenvectors of the covariance
        matrix.
    eigenvectors : (3x3) array
        The eigenvectors of the covariance matrix.
    """
    cov_mat = np.cov(points, rowvar=False)
    eigenvalues, eigenvectors = np.linalg.eig(cov_mat)
    order = np.argsort(-eigenvalues)
    eigenvalues = eigenvalues[order]
    eigenvectors = eigenvectors[:, order]
    return eigenvalues, eigenvectors 
Example 3
Project: FRIDA   Author: LCAV   File: point_cloud.py    MIT License 7 votes vote down vote up
def classical_mds(self, D):
        ''' 
        Classical multidimensional scaling

        Parameters
        ----------
        D : square 2D ndarray
            Euclidean Distance Matrix (matrix containing squared distances between points
        '''

        # Apply MDS algorithm for denoising
        n = D.shape[0]
        J = np.eye(n) - np.ones((n,n))/float(n)
        G = -0.5*np.dot(J, np.dot(D, J))

        s, U = np.linalg.eig(G)

        # we need to sort the eigenvalues in decreasing order
        s = np.real(s)
        o = np.argsort(s)
        s = s[o[::-1]]
        U = U[:,o[::-1]]

        S = np.diag(s)[0:self.dim,:]
        self.X = np.dot(np.sqrt(S),U.T) 
Example 4
Project: building-boundary   Author: Geodan   File: regularize.py    MIT License 6 votes vote down vote up
def sort_orientations(orientations):
    """
    Sort orientations by the length of the segments which have that
    orientation.

    Parameters
    ----------
    orientations : dict
        The orientations and corrisponding lengths

    Returns
    -------
    sorted_orientations : list of float

    """
    unsorted_orientations = [o['orientation'] for o in orientations]
    lengths = [o['size'] for o in orientations]
    sort = np.argsort(lengths)[::-1]
    sorted_orientations = np.array(unsorted_orientations)[sort].tolist()
    return sorted_orientations 
Example 5
Project: claxon   Author: vanatteveldt   File: ml.py    GNU General Public License v3.0 6 votes vote down vote up
def get_todo(session: Session, model: Language, n=10) -> OrderedDict:
    """Populate the queue of documents to code"""
    done = {a.document_id for a in Annotation.objects.filter(document__gold=False, label=session.label)}
    todo = Document.objects.filter(gold=False).exclude(pk__in=done)
    if session.query:
        todo = todo.filter(text__icontains=session.query)
    todo = list(todo.values_list("id", flat=True))
    logging.debug("{ntodo} documents in todo (query: {q}, done={ndone})"
                  .format(ntodo=len(todo), ndone=len(done), q=session.query))
    if len(todo) > settings.N_SAMPLE:
        todo = sample(todo, settings.N_SAMPLE)

    tc = model.get_pipe("textcat")
    tokens = [get_tokens(model, doc_id) for doc_id in todo]
    scores = [d.cats[session.label.label] for d in tc.pipe(tokens)]
    uncertainty = [abs(score - 0.5) for score in scores]
    index = list(argsort(uncertainty))[:n]

    return OrderedDict((todo[i], scores[i]) for i in index) 
Example 6
Project: FRIDA   Author: LCAV   File: doa.py    MIT License 6 votes vote down vote up
def _peaks1D(self):
        if self.num_src == 1:
            self.src_idx[0] = np.argmax(self.P)
            self.sources[:, 0] = self.loc[:, self.src_idx[0]]
            self.phi_recon = self.theta[self.src_idx[0]]
        else:
            peak_idx = []
            n = self.P.shape[0]
            for i in range(self.num_loc):
                # straightforward peak finding
                if self.P[i] >= self.P[(i-1)%n] and self.P[i] > self.P[(i+1)%n]:
                    if len(peak_idx) == 0 or peak_idx[-1] != i-1:
                        if not (i == self.num_loc and self.P[i] == self.P[0]):
                            peak_idx.append(i)

            peaks = self.P[peak_idx]
            max_idx = np.argsort(peaks)[-self.num_src:]
            self.src_idx = [peak_idx[k] for k in max_idx]
            self.sources = self.loc[:, self.src_idx]
            self.phi_recon = self.theta[self.src_idx]
            self.num_src = len(self.src_idx)


# ------------------Miscellaneous Functions---------------------# 
Example 7
Project: skylab   Author: coenders   File: ps_injector.py    GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, logE, logFlux, gamma, deg=4, **kwargs):
        # Make sure that energy bins are of increasing order.
        sorter = np.argsort(logE)
        energy = logE[sorter]
        flux = logFlux[sorter]

        # Make sure that energy bins contain only unique values.
        unique = np.argwhere(np.diff(energy) > 0.)
        energy = energy[unique]
        flux = flux[unique]

        self._spline = scipy.interpolate.InterpolatedUnivariateSpline(
            energy, flux, k=deg)

        # Use default energy range of flux parametrization.
        kwargs.setdefault("e_range", [10.**np.amin(logE), 10.**np.amax(logE)])

        super(ModelInjector, self).__init__(gamma, **kwargs) 
Example 8
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: ocr_predict.py    Apache License 2.0 6 votes vote down vote up
def forward_ocr(self, img_):
        img_ = cv2.resize(img_, (80, 30))
        img_ = img_.transpose(1, 0)
        print(img_.shape)
        img_ = img_.reshape((1, 80, 30))
        print(img_.shape)
        # img_ = img_.reshape((80 * 30))
        img_ = np.multiply(img_, 1 / 255.0)
        self.predictor.forward(data=img_, **self.init_state_dict)
        prob = self.predictor.get_output(0)
        label_list = []
        for p in prob:
            print(np.argsort(p))
            max_index = np.argsort(p)[::-1][0]
            label_list.append(max_index)
        return self.__get_string(label_list) 
Example 9
Project: DensityPeakCluster   Author: lanbing510   File: plot.py    MIT License 6 votes vote down vote up
def plot_rhodelta_rho(rho, delta):
	'''
	Plot scatter diagram for rho*delta_rho points

	Args:
		rho   : rho list
		delta : delta list
	'''
	logger.info("PLOT: rho*delta_rho plot")
	y=rho*delta
	r_index=np.argsort(-y)
	x=np.zeros(y.shape[0])
	idx=0
	for r in r_index:
	    x[r]=idx
	    idx+=1
	plt.figure(2)
	plt.clf()
	plt.scatter(x,y)
	plt.xlabel('sorted rho')
	plt.ylabel('rho*delta')
	plt.title("Decision Graph RhoDelta-Rho")
	plt.show()
	plt.savefig('Decision Graph RhoDelta-Rho.jpg') 
Example 10
Project: DOTA_models   Author: ringringyi   File: utils.py    Apache License 2.0 6 votes vote down vote up
def calc_pr(gt, out, wt=None):
  if wt is None:
    wt = np.ones((gt.size,1))

  gt = gt.astype(np.float64).reshape((-1,1))
  wt = wt.astype(np.float64).reshape((-1,1))
  out = out.astype(np.float64).reshape((-1,1))

  gt = gt*wt
  tog = np.concatenate([gt, wt, out], axis=1)*1.
  ind = np.argsort(tog[:,2], axis=0)[::-1]
  tog = tog[ind,:]
  cumsumsortgt = np.cumsum(tog[:,0])
  cumsumsortwt = np.cumsum(tog[:,1])
  prec = cumsumsortgt / cumsumsortwt
  rec = cumsumsortgt / np.sum(tog[:,0])

  ap = voc_ap(rec, prec)
  return ap, rec, prec 
Example 11
Project: DOTA_models   Author: ringringyi   File: nav_env.py    Apache License 2.0 6 votes vote down vote up
def _get_room_dimensions(file_name, resolution, origin, flip=False):
  if fu.exists(file_name):
    a = utils.load_variables(file_name)['room_dimension']
    names = a.keys()
    dims = np.concatenate(a.values(), axis=0).reshape((-1,6))
    ind = np.argsort(names)
    dims = dims[ind,:]
    names = [names[x] for x in ind]
    if flip:
      dims_new = dims*1
      dims_new[:,1] = -dims[:,4]
      dims_new[:,4] = -dims[:,1]
      dims = dims_new*1

    dims = dims*100.
    dims[:,0] = dims[:,0] - origin[0]
    dims[:,1] = dims[:,1] - origin[1]
    dims[:,3] = dims[:,3] - origin[0]
    dims[:,4] = dims[:,4] - origin[1]
    dims = dims / resolution
    out = {'names': names, 'dims': dims}
  else:
    out = None
  return out 
Example 12
Project: nlimb   Author: cbschaff   File: component_chopper.py    MIT License 6 votes vote down vote up
def chop(self):
        n = self.components_left()
        if n <= 1: return

        if self.rank == 0:
            avg_component_reward = np.zeros_like(self.logmixprobs)
            for i,c in enumerate(self.logmixprobs):
                if c != 0: continue
                rews = []
                for _ in range(100):
                    robot = self.sampler.sample_gaussian(i)[0]
                    rews.append(self.test_robot(robot))
                avg_component_reward[i] = np.mean(rews)
            inds = np.argsort(avg_component_reward)
            for ind in inds:
                if self.components_left() <= n // 2:
                    break
                self.logmixprobs[ind] = -1000000.

            self.tflogmixprobs.load(self.logmixprobs)
        self.sync() 
Example 13
Project: pidforest   Author: vatsalsharan   File: forest.py    MIT License 6 votes vote down vote up
def predict(self, pts, err=0.1, pct=50):
        _, n_pts = np.shape(pts)
        scores = np.zeros((self.n_trees, n_pts))  # need to do this streaming
        indices = [i for i in range(n_pts)]
        for i in range(self.n_trees):
            self.tree[i].compute_split(pts, indices, scores[i])
        n_err = int(err * n_pts)
        min_score = np.percentile(scores, pct, axis=0)
        top_indices = np.argsort(min_score)[:n_err]
        anom_pts = {}
        anom_scores = {}
        anom_pct = {}
        for i in range(n_err):
            anom_pts[top_indices[i]] = pts[:, top_indices[i]]
            anom_scores[top_indices[i]] = scores[:, top_indices[i]]
            anom_pct[top_indices[i]] = min_score[top_indices[i]]
        return top_indices, anom_pts, anom_scores, anom_pct, min_score 
Example 14
Project: pidforest   Author: vatsalsharan   File: forest.py    MIT License 6 votes vote down vote up
def predict_depth(self, pts, err=0.1, pct=50):
        _, n_pts = np.shape(pts)
        scores = np.zeros((self.n_trees, n_pts))  # need to do this streaming
        indices = [i for i in range(n_pts)]
        for i in range(self.n_trees):
            self.tree[i].compute_depth(pts, indices, scores[i])
        n_err = int(err * n_pts)
        min_score = np.percentile(scores, pct, axis=0)
        top_indices = np.argsort(min_score)[:n_err]
        anom_pts = {}
        anom_scores = {}
        anom_pct = {}
        for i in range(n_err):
            anom_pts[top_indices[i]] = pts[:, top_indices[i]]
            anom_scores[top_indices[i]] = scores[:, top_indices[i]]
            anom_pct[top_indices[i]] = min_score[top_indices[i]]
        return top_indices, anom_pts, anom_scores, anom_pct, min_score 
Example 15
Project: pidforest   Author: vatsalsharan   File: forest.py    MIT License 6 votes vote down vote up
def find_min_cube(self, pts, n_err=1, pct1=50, pct2=50):
        _, n_pts = np.shape(pts)
        scores = np.zeros((self.n_trees, n_pts))  # need to do this streaming
        indices = [i for i in range(n_pts)]
        for i in range(self.n_trees):
            self.tree[i].compute_split(pts, indices, scores[i])
        min_score = np.percentile(scores, pct1, axis=0)
        top_indices = np.argsort(min_score)[:n_err]
        sort_trees = np.argsort(scores, axis=0)
        min_tree = sort_trees[int(self.n_trees*pct2/100.0)]
        boxes = []
        for i in range(0,n_err):
            print('----')
            best_tree = min_tree[top_indices[i]]
            boxes.append( self.tree[best_tree].compute_box(pts[:,top_indices[i]]) )
            # boxes = self.tree[best_tree].compute_box(pts[:,top_indices[i]])
        return top_indices, boxes 
Example 16
Project: pidforest   Author: vatsalsharan   File: new_forest.py    MIT License 6 votes vote down vote up
def predict(self, pts, err=0.1, pct=50):
        _, n_pts = np.shape(pts)
        scores = np.zeros((self.n_trees, n_pts))  # need to do this streaming
        indices = np.arange(n_pts)
        for i in range(self.n_trees):
            self.tree[i].compute_split(pts, indices, scores[i])
        n_err = int(err * n_pts)
        min_score = np.percentile(scores, pct, axis=0)
        top_indices = np.argsort(min_score)[:n_err]
        anom_pts = {}
        anom_scores = {}
        anom_pct = {}
        for i in range(n_err):
            anom_pts[top_indices[i]] = pts[:, top_indices[i]]
            anom_scores[top_indices[i]] = scores[:, top_indices[i]]
            anom_pct[top_indices[i]] = min_score[top_indices[i]]
        return top_indices, anom_pts, anom_scores, anom_pct 
Example 17
Project: pidforest   Author: vatsalsharan   File: myforest.py    MIT License 6 votes vote down vote up
def predict(self, pts, err=0.1, pct=50):
        _, n_pts = np.shape(pts)
        scores = np.zeros((self.n_trees, n_pts))  # need to do this streaming
        indices = [i for i in range(n_pts)]
        for i in range(self.n_trees):
            self.tree[i].compute_split(pts, indices, scores[i])
        n_err = int(err * n_pts)
        min_score = np.percentile(scores, pct, axis=0)
        top_indices = np.argsort(min_score)[:n_err]
        anom_pts = {}
        anom_scores = {}
        anom_pct = {}
        for i in range(n_err):
            anom_pts[top_indices[i]] = pts[:, top_indices[i]]
            anom_scores[top_indices[i]] = scores[:, top_indices[i]]
            anom_pct[top_indices[i]] = min_score[top_indices[i]]
        return top_indices, anom_pts, anom_scores, anom_pct 
Example 18
Project: lung_nodule_classifier   Author: xairc   File: layers.py    MIT License 6 votes vote down vote up
def nms(output, nms_th):
    if len(output) == 0:
        return output

    output = output[np.argsort(-output[:, 0])]
    bboxes = [output[0]]
    
    for i in np.arange(1, len(output)):
        bbox = output[i]
        flag = 1
        for j in range(len(bboxes)):
            if iou(bbox[1:5], bboxes[j][1:5]) >= nms_th:
                flag = -1
                break
        if flag == 1:
            bboxes.append(bbox)
    
    bboxes = np.asarray(bboxes, np.float32)
    return bboxes 
Example 19
Project: relay-bench   Author: uwsampl   File: run.py    Apache License 2.0 6 votes vote down vote up
def check_results(graph_module, remote, vta_env):
    """Verify the input was classified correctly."""
    # Get classification results.
    tvm_output = graph_module.get_output(
        0, tvm.nd.empty((vta_env.BATCH, 1000), 'float32', remote.cpu(0)))
    top_categories = np.argsort(tvm_output.asnumpy()[0])

    # Load ImageNet categories.
    with open(os.path.join(os.path.dirname(__file__), 'synset.txt')) as f:
        # "synset.txt" is formatted as a Python dict, so we eval the file
        # contents to get a Python dict.
        synset = eval(f.read())

    # This just checks that one of the 5 top categories is one variety of cat;
    # this is by no means an accurate assessment of how quantization affects
    # classification accuracy but is meant to catch changes to the quantization
    # pass that would accuracy in the CI.
    cat_detected = False
    for k in top_categories[-5:]:
        if 'cat' in synset[k]:
            cat_detected = True
    assert(cat_detected) 
Example 20
Project: transferlearning   Author: jindongwang   File: TCA.py    MIT License 6 votes vote down vote up
def fit(self, Xs, Xt):
        '''
        Transform Xs and Xt
        :param Xs: ns * n_feature, source feature
        :param Xt: nt * n_feature, target feature
        :return: Xs_new and Xt_new after TCA
        '''
        X = np.hstack((Xs.T, Xt.T))
        X /= np.linalg.norm(X, axis=0)
        m, n = X.shape
        ns, nt = len(Xs), len(Xt)
        e = np.vstack((1 / ns * np.ones((ns, 1)), -1 / nt * np.ones((nt, 1))))
        M = e * e.T
        M = M / np.linalg.norm(M, 'fro')
        H = np.eye(n) - 1 / n * np.ones((n, n))
        K = kernel(self.kernel_type, X, None, gamma=self.gamma)
        n_eye = m if self.kernel_type == 'primal' else n
        a, b = np.linalg.multi_dot([K, M, K.T]) + self.lamb * np.eye(n_eye), np.linalg.multi_dot([K, H, K.T])
        w, V = scipy.linalg.eig(a, b)
        ind = np.argsort(w)
        A = V[:, ind[:self.dim]]
        Z = np.dot(A.T, K)
        Z /= np.linalg.norm(Z, axis=0)
        Xs_new, Xt_new = Z[:, :ns].T, Z[:, ns:].T
        return Xs_new, Xt_new 
Example 21
Project: wikilinks   Author: trovdimi   File: fitpowerlaw.py    MIT License 5 votes vote down vote up
def giniCoeff(net_name, net):
    # requires all values in x to be zero or positive numbers,
    # otherwise results are undefined
    degree_dist={}
    for v in net.vertices():
        degree_dist[v] = v.out_degree() + v.in_degree()
    x = np.asarray(degree_dist.values())
    n = len(x)
    s = x.sum()
    r = np.argsort(np.argsort(-x)) # calculates zero-based ranks
    gini = 1 - (2.0 * (r*x).sum() + s)/(n*s)
    print 'gini', str(gini)

    degree_dist={}
    for v in net.vertices():
        degree_dist[v] =v.in_degree()
    x = np.asarray(degree_dist.values())
    n = len(x)
    s = x.sum()
    r = np.argsort(np.argsort(-x)) # calculates zero-based ranks
    gini = 1 - (2.0 * (r*x).sum() + s)/(n*s)
    print "in"
    print 'gini', str(gini)

    degree_dist={}
    for v in net.vertices():
        degree_dist[v] =v.out_degree()
    x = np.asarray(degree_dist.values())
    n = len(x)
    s = x.sum()
    r = np.argsort(np.argsort(-x)) # calculates zero-based ranks
    gini = 1 - (2.0 * (r*x).sum() + s)/(n*s)
    print "out"
    print 'gini', str(gini)
    return gini 
Example 22
Project: Automated-Social-Annotation   Author: acadTags   File: BiGRU_train.py    MIT License 5 votes vote down vote up
def get_label_using_logits(logits,vocabulary_index2word_label,top_number=1):
    #print("get_label_using_logits.logits:",logits) #1-d array: array([-5.69036102, -8.54903221, -5.63954401, ..., -5.83969498,-5.84496021, -6.13911009], dtype=float32))
    index_list=np.argsort(logits)[-top_number:]
    index_list=index_list[::-1]
    #label_list=[]
    #for index in index_list:
    #    label=vocabulary_index2word_label[index]
    #    label_list.append(label) #('get_label_using_logits.label_list:', [u'-3423450385060590478', u'2838091149470021485', u'-3174907002942471215', u'-1812694399780494968', u'6815248286057533876'])
    return index_list

#从sigmoid(logits)中取出大于0.5的 
Example 23
Project: Automated-Social-Annotation   Author: acadTags   File: HAN_train.py    MIT License 5 votes vote down vote up
def get_label_using_logits(logits,vocabulary_index2word_label,top_number=1):
    #print("get_label_using_logits.logits:",logits) #1-d array: array([-5.69036102, -8.54903221, -5.63954401, ..., -5.83969498,-5.84496021, -6.13911009], dtype=float32))
    index_list=np.argsort(logits)[-top_number:]
    index_list=index_list[::-1]
    #label_list=[]
    #for index in index_list:
    #    label=vocabulary_index2word_label[index]
    #    label_list.append(label) #('get_label_using_logits.label_list:', [u'-3423450385060590478', u'2838091149470021485', u'-3174907002942471215', u'-1812694399780494968', u'6815248286057533876'])
    return index_list

#从sigmoid(logits)中取出大于0.5的 
Example 24
Project: Automated-Social-Annotation   Author: acadTags   File: LDA.py    MIT License 5 votes vote down vote up
def get_doc_ind_from_vec(vec,k_num_doc=1):
    index_list=np.argsort(vec)[-k_num_doc:]
    index_list=index_list[::-1]
    return index_list 
Example 25
Project: Automated-Social-Annotation   Author: acadTags   File: JMAN_train.py    MIT License 5 votes vote down vote up
def get_label_using_logits(logits,vocabulary_index2word_label,top_number=1):
    #print("get_label_using_logits.logits:",logits) #1-d array: array([-5.69036102, -8.54903221, -5.63954401, ..., -5.83969498,-5.84496021, -6.13911009], dtype=float32))
    index_list=np.argsort(logits)[-top_number:]
    index_list=index_list[::-1]
    #label_list=[]
    #for index in index_list:
    #    label=vocabulary_index2word_label[index]
    #    label_list.append(label) #('get_label_using_logits.label_list:', [u'-3423450385060590478', u'2838091149470021485', u'-3174907002942471215', u'-1812694399780494968', u'6815248286057533876'])
    return index_list

#get label from logits with threshold 
Example 26
Project: Lane-And-Vehicle-Detection   Author: JustinHeaton   File: main.py    MIT License 5 votes vote down vote up
def sort_vals(self, xvals, yvals):
        sorted_index = np.argsort(yvals)
        sorted_yvals = yvals[sorted_index]
        sorted_xvals = xvals[sorted_index]
        return sorted_xvals, sorted_yvals 
Example 27
Project: mmdetection   Author: open-mmlab   File: recall.py    Apache License 2.0 5 votes vote down vote up
def eval_recalls(gts,
                 proposals,
                 proposal_nums=None,
                 iou_thrs=None,
                 print_summary=True):
    """Calculate recalls.

    Args:
        gts(list or ndarray): a list of arrays of shape (n, 4)
        proposals(list or ndarray): a list of arrays of shape (k, 4) or (k, 5)
        proposal_nums(int or list of int or ndarray): top N proposals
        thrs(float or list or ndarray): iou thresholds

    Returns:
        ndarray: recalls of different ious and proposal nums
    """

    img_num = len(gts)
    assert img_num == len(proposals)

    proposal_nums, iou_thrs = set_recall_param(proposal_nums, iou_thrs)

    all_ious = []
    for i in range(img_num):
        if proposals[i].ndim == 2 and proposals[i].shape[1] == 5:
            scores = proposals[i][:, 4]
            sort_idx = np.argsort(scores)[::-1]
            img_proposal = proposals[i][sort_idx, :]
        else:
            img_proposal = proposals[i]
        prop_num = min(img_proposal.shape[0], proposal_nums[-1])
        if gts[i] is None or gts[i].shape[0] == 0:
            ious = np.zeros((0, img_proposal.shape[0]), dtype=np.float32)
        else:
            ious = bbox_overlaps(gts[i], img_proposal[:prop_num, :4])
        all_ious.append(ious)
    all_ious = np.array(all_ious)
    recalls = _recalls(all_ious, proposal_nums, iou_thrs)
    if print_summary:
        print_recall_summary(recalls, proposal_nums, iou_thrs)
    return recalls 
Example 28
Project: models   Author: kipoi   File: gather.py    MIT License 5 votes vote down vote up
def average_labranchor(df, model_name, col_types):
    import numpy as np
    # choose the maximum diff
    diff_cols = df.columns.values[df.columns.astype(str).str.contains("DIFF")]
    model_outputs = [int(el.split("_")[-1]) for el in diff_cols]
    model_outputs_order = np.argsort(model_outputs)
    # select the model output tha gives the maximum absolute difference
    max_col_id = df[diff_cols[model_outputs_order]].abs().values.argmax(axis=1)
    #
    # just to be sure it will work:
    assert np.all(df[diff_cols[model_outputs_order]].abs().values[np.arange(len(max_col_id)), max_col_id] == df[diff_cols].abs().max(axis=1).values)
    #
    averaged = {}
    usable_columns = df.columns.tolist()
    for ct in col_types:
        col_sel = [col for col in usable_columns if ct in col]
        usable_columns = [col for col in usable_columns if col not in col_sel]
        if len(col_sel) == 0:
            continue
        # average
        model_outputs = [int(el.split("_")[-1]) for el in col_sel]
        model_outputs_order = np.argsort(model_outputs)
        # use the column selection from before
        keep_vals = df[np.array(col_sel)[model_outputs_order]].values[np.arange(len(max_col_id)), max_col_id]
        averaged[model_name + ct.lower()] = keep_vals
    #
    return pd.DataFrame(averaged, index=df.index) 
Example 29
Project: models   Author: kipoi   File: gather.py    MIT License 5 votes vote down vote up
def average_labranchor(df, model_name, col_types):
    import numpy as np
    # choose the maximum diff
    diff_cols = df.columns.values[df.columns.astype(str).str.contains("DIFF")]
    model_outputs = [int(el.split("_")[-1]) for el in diff_cols]
    model_outputs_order = np.argsort(model_outputs)
    # select the model output tha gives the maximum absolute difference
    max_col_id = df[diff_cols[model_outputs_order]].abs().values.argmax(axis=1)
    #
    # just to be sure it will work:
    assert np.all(df[diff_cols[model_outputs_order]].abs().values[np.arange(len(max_col_id)), max_col_id] == df[diff_cols].abs().max(axis=1).values)
    #
    averaged = {}
    usable_columns = df.columns.tolist()
    for ct in col_types:
        col_sel = [col for col in usable_columns if ct in col]
        usable_columns = [col for col in usable_columns if col not in col_sel]
        if len(col_sel) == 0:
            continue
        # average
        model_outputs = [int(el.split("_")[-1]) for el in col_sel]
        model_outputs_order = np.argsort(model_outputs)
        # use the column selection from before
        keep_vals = df[np.array(col_sel)[model_outputs_order]].values[np.arange(len(max_col_id)), max_col_id]
        averaged[model_name + ct.lower()] = keep_vals
    #
    return pd.DataFrame(averaged, index=df.index) 
Example 30
Project: Image-Caption-Generator   Author: dabasajay   File: model.py    MIT License 5 votes vote down vote up
def generate_caption_beam_search(model, tokenizer, image, max_length, beam_index=3):
	# in_text --> [[idx,prob]] ;prob=0 initially
	in_text = [[tokenizer.texts_to_sequences(['startseq'])[0], 0.0]]
	while len(in_text[0][0]) < max_length:
		tempList = []
		for seq in in_text:
			padded_seq = pad_sequences([seq[0]], maxlen=max_length)
			preds = model.predict([image,padded_seq], verbose=0)
			# Take top (i.e. which have highest probailities) `beam_index` predictions
			top_preds = np.argsort(preds[0])[-beam_index:]
			# Getting the top `beam_index` predictions and 
			for word in top_preds:
				next_seq, prob = seq[0][:], seq[1]
				next_seq.append(word)
				# Update probability
				prob += preds[0][word]
				# Append as input for generating the next word
				tempList.append([next_seq, prob])
		in_text = tempList
		# Sorting according to the probabilities
		in_text = sorted(in_text, reverse=False, key=lambda l: l[1])
		# Take the top words
		in_text = in_text[-beam_index:]
	in_text = in_text[-1][0]
	final_caption_raw = [int_to_word(i,tokenizer) for i in in_text]
	final_caption = []
	for word in final_caption_raw:
		if word=='endseq':
			break
		else:
			final_caption.append(word)
	final_caption.append('endseq')
	return ' '.join(final_caption) 
Example 31
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: metric.py    Apache License 2.0 5 votes vote down vote up
def update(self, labels, preds):
        """Updates the internal evaluation result.

        Parameters
        ----------
        labels : list of `NDArray`
            The labels of the data.

        preds : list of `NDArray`
            Predicted values.
        """
        labels, preds = check_label_shapes(labels, preds, True)

        for label, pred_label in zip(labels, preds):
            assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims'
            pred_label = numpy.argsort(pred_label.asnumpy().astype('float32'), axis=1)
            label = label.asnumpy().astype('int32')
            check_label_shapes(label, pred_label)
            num_samples = pred_label.shape[0]
            num_dims = len(pred_label.shape)
            if num_dims == 1:
                self.sum_metric += (pred_label.flat == label.flat).sum()
            elif num_dims == 2:
                num_classes = pred_label.shape[1]
                top_k = min(num_classes, self.top_k)
                for j in range(top_k):
                    self.sum_metric += (pred_label[:, num_classes - 1 - j].flat == label.flat).sum()
            self.num_inst += num_samples 
Example 32
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_operator.py    Apache License 2.0 5 votes vote down vote up
def _syevd_forward(a):
    lam, ut = np.linalg.eig(a)
    ind = np.argsort(lam)
    lam = lam[ind]
    u = ut[:, ind].T
    for i in range(0, a.shape[0]):
        _syevd_forw_eigvec_sign(u[i])
    return u, lam 
Example 33
Project: DensityPeakCluster   Author: lanbing510   File: cluster.py    MIT License 5 votes vote down vote up
def min_distance(max_id, max_dis, distances, rho):
	'''
	Compute all points' min distance to the higher local density point(which is the nearest neighbor)

	Args:
		max_id    : max continues id
		max_dis   : max distance for all points
		distances : distance dict
		rho       : local density vector that index is the point index that start from 1
	
	Returns:
	    min_distance vector, nearest neighbor vector
	'''
	logger.info("PROGRESS: compute min distance to nearest higher density neigh")
	sort_rho_idx = np.argsort(-rho)
	delta, nneigh = [0.0] + [float(max_dis)] * (len(rho) - 1), [0] * len(rho)
	delta[sort_rho_idx[0]] = -1.
	for i in xrange(1, max_id):
		for j in xrange(0, i):
			old_i, old_j = sort_rho_idx[i], sort_rho_idx[j]
			if distances[(old_i, old_j)] < delta[old_i]:
				delta[old_i] = distances[(old_i, old_j)]
				nneigh[old_i] = old_j
		if i % (max_id / 10) == 0:
			logger.info("PROGRESS: at index #%i" % (i))
	delta[sort_rho_idx[0]] = max(delta)
	return np.array(delta, np.float32), np.array(nneigh, np.float32) 
Example 34
Project: Logo-Retrieval-in-Commercial-Plaza   Author: zhang-rongchen   File: logo.py    MIT License 5 votes vote down vote up
def search_img(self,query):
        # read in indexed images' feature vectors and corresponding image names
        print("--------------------------------------------------")
        print("               searching starts")
        print("--------------------------------------------------")

        # extract query image's feature, compute simlarity score and sort
        t2 = time.time()
        img = cv2.imread(query)
        image, logos = self.yolo.detect_image(img)

        for logo in logos:
            t4 = time.time()
            queryVec = self.model.extract_feat(logo)
            scores = np.dot(queryVec, self.feats.T)
            rank_ID = np.argsort(scores)[::-1]
            rank_score = scores[rank_ID]
            t5 = time.time() - t4
            print('t5=' + str(t5))
            print("-------------------------------------------")
            print(rank_ID)
            print(rank_score)
            print("-------------------------------------------")

            # number of top retrieved images to show
            imlist = [self.imgNames[index] for index in rank_ID[0:1]]
            file ='database/'+str(imlist)[3:-2]
            t3 = time.time() - t2
            print('t3=' + str(t3))
            print(rank_score[0])
            return file, t3, rank_score[0] 
Example 35
Project: DOTA_models   Author: ringringyi   File: np_box_list_ops.py    Apache License 2.0 5 votes vote down vote up
def sort_by_field(boxlist, field, order=SortOrder.DESCEND):
  """Sort boxes and associated fields according to a scalar field.

  A common use case is reordering the boxes according to descending scores.

  Args:
    boxlist: BoxList holding N boxes.
    field: A BoxList field for sorting and reordering the BoxList.
    order: (Optional) 'descend' or 'ascend'. Default is descend.

  Returns:
    sorted_boxlist: A sorted BoxList with the field in the specified order.

  Raises:
    ValueError: if specified field does not exist or is not of single dimension.
    ValueError: if the order is not either descend or ascend.
  """
  if not boxlist.has_field(field):
    raise ValueError('Field ' + field + ' does not exist')
  if len(boxlist.get_field(field).shape) != 1:
    raise ValueError('Field ' + field + 'should be single dimension.')
  if order != SortOrder.DESCEND and order != SortOrder.ASCEND:
    raise ValueError('Invalid sort order')

  field_to_sort = boxlist.get_field(field)
  sorted_indices = np.argsort(field_to_sort)
  if order == SortOrder.DESCEND:
    sorted_indices = sorted_indices[::-1]
  return gather(boxlist, sorted_indices) 
Example 36
Project: pypriv   Author: soeaver   File: misc.py    MIT License 5 votes vote down vote up
def nms(dets, thresh):
    if dets.shape[0] == 0:
        return []
    """Pure Python NMS baseline."""
    x1 = dets[:, 0]
    y1 = dets[:, 1]
    x2 = dets[:, 2]
    y2 = dets[:, 3]
    scores = dets[:, 4]

    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        i = order[0]
        keep.append(i)
        xx1 = np.maximum(x1[i], x1[order[1:]])
        yy1 = np.maximum(y1[i], y1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])
        yy2 = np.minimum(y2[i], y2[order[1:]])

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = w * h
        ovr = inter / (areas[i] + areas[order[1:]] - inter)

        inds = np.where(ovr <= thresh)[0]
        order = order[inds + 1]

    return keep 
Example 37
Project: pypriv   Author: soeaver   File: misc.py    MIT License 5 votes vote down vote up
def objs_sort_by_center(objs, target=0):
    """target=0, sort by x; target=1, sort by y;."""
    sorted = []
    centers = []
    for i in objs:
        if target == 0:
            centers.append((i['bbox'][0] + i['bbox'][2]) / 2.0)
        elif target == 1:
            centers.append((i['bbox'][1] + i['bbox'][3]) / 2.0)
    centers_idx = np.argsort(np.asarray(centers))

    for i in centers_idx:
        sorted.append(objs[i])
        
    return sorted 
Example 38
Project: graph-neural-networks   Author: alelab-upenn   File: graphTools.py    GNU General Public License v3.0 5 votes vote down vote up
def computeGFT(S, order = 'no'):
    """
    computeGFT: Computes the frequency basis (eigenvectors) and frequency
        coefficients (eigenvalues) of a given GSO

    Input:

        S (np.array): graph shift operator matrix
        order (string): 'no', 'increasing', 'totalVariation' chosen order of
            frequency coefficients (default: 'no')

    Output:

        E (np.array): diagonal matrix with the frequency coefficients
            (eigenvalues) in the diagonal
        V (np.array): matrix with frequency basis (eigenvectors)
    """
    # Check the correct order input
    assert order == 'totalVariation' or order == 'no' or order == 'increasing'
    # Check the matrix is square
    assert S.shape[0] == S.shape[1]
    # Check if it is symmetric
    symmetric = np.allclose(S, S.T, atol = zeroTolerance)
    # Then, compute eigenvalues and eigenvectors
    if symmetric:
        e, V = np.linalg.eigh(S)
    else:
        e, V = np.linalg.eig(S)
    # Sort the eigenvalues by the desired error:
    if order == 'totalVariation':
        eMax = np.max(e)
        sortIndex = np.argsort(np.abs(e - eMax))
    elif order == 'increasing':
        sortIndex = np.argsort(np.abs(e))
    else:
        sortIndex = np.arange(0, S.shape[0])
    e = e[sortIndex]
    V = V[:, sortIndex]
    E = np.diag(e)
    return E, V 
Example 39
Project: graph-neural-networks   Author: alelab-upenn   File: graphTools.py    GNU General Public License v3.0 5 votes vote down vote up
def computeSourceNodes(A, C):
    """
    computeSourceNodes: compute source nodes for the source localization problem
    
    Input:
        A (np.array): adjacency matrix of shape N x N
        C (int): number of classes
        
    Output:
        sourceNodes (list): contains the indices of the C source nodes
        
    Uses the adjacency matrix to compute C communities by means of spectral 
    clustering, and then selects the node with largest degree within each 
    community
    """
    sourceNodes = []
    degree = np.sum(A, axis = 0) # degree of each vector
    # Compute communities
    communityClusters = SpectralClustering(n_clusters = C,
                                           affinity = 'precomputed',
                                           assign_labels = 'discretize')
    communityClusters = communityClusters.fit(A)
    communityLabels = communityClusters.labels_
    # For each community
    for c in range(C):
        communityNodes = np.nonzero(communityLabels == c)[0]
        degreeSorted = np.argsort(degree[communityNodes])
        sourceNodes = sourceNodes + [communityNodes[degreeSorted[-1]]]
    
    return sourceNodes 
Example 40
Project: graph-neural-networks   Author: alelab-upenn   File: graphTools.py    GNU General Public License v3.0 5 votes vote down vote up
def permDegree(S):
    """
    permDegree: determines the permutation by degree (nodes ordered from highest
        degree to lowest)

    Input:
        S (np.array): matrix

    Output:
        permS (np.array): matrix permuted
        order (list): list of indices to permute S to turn into permS.
    """
    assert len(S.shape) == 2 or len(S.shape) == 3
    if len(S.shape) == 2:
        assert S.shape[0] == S.shape[1]
        S = S.reshape([1, S.shape[0], S.shape[1]])
        scalarWeights = True
    else:
        assert S.shape[1] == S.shape[2]
        scalarWeights = False
    # Compute the degree
    d = np.sum(np.sum(S, axis = 1), axis = 0)
    # Sort ascending order (from min degree to max degree)
    order = np.argsort(d)
    # Reverse sorting
    order = np.flip(order,0)
    # And update S
    S = S[:,order,:][:,:,order]
    # If the original GSO assumed scalar weights, get rid of the extra dimension
    if scalarWeights:
        S = S.reshape([S.shape[1], S.shape[2]])

    return S, order.tolist() 
Example 41
Project: graph-neural-networks   Author: alelab-upenn   File: graphTools.py    GNU General Public License v3.0 5 votes vote down vote up
def perm_adjacency(A, indices):
    # Function written by M. Defferrard, taken verbatim, from 
    # https://github.com/mdeff/cnn_graph/blob/master/lib/coarsening.py#L242
    """
    Permute adjacency matrix, i.e. exchange node ids,
    so that binary unions form the clustering tree.
    """
    if indices is None:
        return A

    M, M = A.shape
    Mnew = len(indices)
    assert Mnew >= M
    A = A.tocoo()

    # Add Mnew - M isolated vertices.
    if Mnew > M:
        rows = scipy.sparse.coo_matrix((Mnew-M,    M), dtype=np.float32)
        cols = scipy.sparse.coo_matrix((Mnew, Mnew-M), dtype=np.float32)
        A = scipy.sparse.vstack([A, rows])
        A = scipy.sparse.hstack([A, cols])

    # Permute the rows and the columns.
    perm = np.argsort(indices)
    A.row = np.array(perm)[A.row]
    A.col = np.array(perm)[A.col]

    # assert np.abs(A - A.T).mean() < 1e-9
    assert type(A) is scipy.sparse.coo.coo_matrix
    return A 
Example 42
Project: gullikson-scripts   Author: kgullikson88   File: HelperFunctions.py    MIT License 5 votes vote down vote up
def GetSurrounding(full_list, value, return_index=False):
    """
    Takes a list and a value, and returns the two list elements
      closest to the value
    If return_index is True, it will return the index of the surrounding
      elements rather than the elements themselves
    """
    sorter = np.argsort(full_list)
    full_list = sorted(full_list)
    closest = np.argmin([abs(v - value) for v in full_list])
    next_best = closest - 1 if full_list[closest] > value or closest == len(full_list) - 1 else closest + 1
    if return_index:
        return sorter[closest], sorter[next_best]
    else:
        return full_list[closest], full_list[next_best] 
Example 43
Project: gullikson-scripts   Author: kgullikson88   File: CCF_Systematics.py    MIT License 5 votes vote down vote up
def predict(self, x, N=100, highest=False):
        """
        Predict the y value for the given x values.
        """
        if self.sampler is None:
            logging.warn('Need to run the fit method before predict!')
            return

        # Find the N best walkers
        if N == 'all':
            N = self.sampler.flatchain.shape[0]
        else:
            N = min(N, self.sampler.flatchain.shape[0])
        
        if highest:
            indices = np.argsort(self.sampler.flatlnprobability)[:N]
            pars = self.sampler.flatchain[indices]
        else:
            pars = self.sampler.flatchain[:N]

        yvals = []
        for i, p in enumerate(pars):
            logging.info('Generating GP samples for iteration {}/{}'.format(i+1, len(pars)))
            a, tau = np.exp(p[:2])
            gp = george.GP(a * kernels.ExpSquaredKernel(tau))
            gp.compute(self.x, self.yerr)
            s = gp.sample_conditional(self.y - self.x, x) + x
            yvals.append(s)

        return np.array(yvals) 
Example 44
Project: ARPET   Author: juliagarriga   File: util.py    MIT License 5 votes vote down vote up
def find_nearest_keypoints(keypoints, pos, number=1):
	if type(pos) is tuple:
		pos = np.array(pos)
	if type(keypoints) is list:
		keypoints = keypoints_cv_to_np(keypoints)

	pos_to_keypoints = np.sqrt(np.power(keypoints - pos, 2).sum(axis=1))
	ind = np.argsort(pos_to_keypoints)
	return ind[:number] 
Example 45
Project: whatsapp-stats   Author: nielsrolf   File: analyze.py    Apache License 2.0 5 votes vote down vote up
def per_hour_plot(group, stat, keys, values):
    perm = np.argsort(keys)
    actual_categorical_plot(group, stat, np.array(keys)[perm], np.array(values)[perm]) 
Example 46
Project: rasa-faq-bot   Author: nghuyong   File: actions.py    MIT License 5 votes vote down vote up
def get_most_similar_standard_question_id(self, query_question):
        query_vector = self.bc.encode([query_question])[0]
        score = np.sum((self.standard_questions_encoder * query_vector), axis=1) / (
                self.standard_questions_encoder_len * (np.sum(query_vector * query_vector) ** 0.5))
        top_id = np.argsort(score)[::-1][0]
        return top_id, score[top_id] 
Example 47
Project: sitq   Author: shiroyagicorp   File: tests_mips.py    MIT License 5 votes vote down vote up
def _brute_force(items, query):
    return np.argsort(items.dot(query))[::-1] 
Example 48
Project: lung_nodule_classifier   Author: xairc   File: layers.py    MIT License 5 votes vote down vote up
def topkpbb(pbb,lbb,nms_th,detect_th,topk=30):
    conf_th = 0
    fp = []
    tp = []
    while len(tp)+len(fp)<topk:
        conf_th = conf_th-0.2
        tp, fp, fn, _ = acc(pbb, lbb, conf_th, nms_th, detect_th)
        if conf_th<-3:
            break
    tp = np.array(tp).reshape([len(tp),6])
    fp = np.array(fp).reshape([len(fp),6])
    fn = np.array(fn).reshape([len(fn),5])
    allp  = np.concatenate([tp,fp],0)
    sorting = np.argsort(allp[:,0])[::-1]
    n_tp = len(tp)
    topk = np.min([topk,len(allp)])
    tp_in_topk = np.array([i for i in range(n_tp) if i in sorting[:topk]])
    fp_in_topk = np.array([i for i in range(topk) if sorting[i] not in range(n_tp)])
#     print(fp_in_topk)
    fn_i =       np.array([i for i in range(n_tp) if i not in sorting[:topk]])
    newallp = allp[:topk]
    if len(fn_i)>0:
        fn = np.concatenate([fn,tp[fn_i,:5]])
    else:
        fn = fn
    if len(tp_in_topk)>0:
        tp = tp[tp_in_topk]
    else:
        tp = []
    if len(fp_in_topk)>0:
        fp = newallp[fp_in_topk]
    else:
        fp = []
    return tp, fp , fn 
Example 49
Project: fine-lm   Author: akzaidi   File: wiki.py    MIT License 5 votes vote down vote up
def scramble(self, seq):
    seq = np.array(seq)
    num_permute = int(self.sequence_length * self.scramble_fraction)
    full_permutation = np.random.permutation(self.sequence_length)
    inverse_full_permutation = np.argsort(full_permutation)
    partial_permutation = np.random.permutation(num_permute)
    seq = seq[full_permutation]
    seq = np.concatenate(
        (seq[:num_permute][partial_permutation], seq[num_permute:]))
    seq = seq[inverse_full_permutation]
    seq = list(seq)
    return seq 
Example 50
Project: fine-lm   Author: akzaidi   File: train.py    MIT License 5 votes vote down vote up
def add_noise_python(words, dropout=0.1, k=3):
  """Applies the noise model in input words.

  Args:
    words: A numpy vector of word ids.
    dropout: The probability to drop words.
    k: Maximum distance of the permutation.

  Returns:
    A noisy numpy vector of word ids.
  """

  def _drop_words(words, probability):
    """Drops words with the given probability."""
    length = len(words)
    keep_prob = np.random.uniform(size=length)
    keep = np.random.uniform(size=length) > probability
    if np.count_nonzero(keep) == 0:
      ind = np.random.randint(0, length)
      keep[ind] = True
    words = np.take(words, keep.nonzero())[0]
    return words

  def _rand_perm_with_constraint(words, k):
    """Randomly permutes words ensuring that words are no more than k positions
    away from their original position."""
    length = len(words)
    offset = np.random.uniform(size=length) * (k + 1)
    new_pos = np.arange(length) + offset
    return np.take(words, np.argsort(new_pos))

  words = _drop_words(words, dropout)
  words = _rand_perm_with_constraint(words, k)
  return words 
Example 51
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: roidb.py    MIT License 5 votes vote down vote up
def rank_roidb_ratio(roidb):
    # rank roidb based on the ratio between width and height.
    ratio_large = 2  # largest ratio to preserve.
    ratio_small = 0.5  # smallest ratio to preserve.

    ratio_list = []
    for i in range(len(roidb)):
        width = roidb[i]['width']
        height = roidb[i]['height']
        ratio = width / float(height)

        if cfg.TRAIN.ASPECT_CROPPING:
            if ratio > ratio_large:
                roidb[i]['need_crop'] = 1
                ratio = ratio_large
            elif ratio < ratio_small:
                roidb[i]['need_crop'] = 1
                ratio = ratio_small
            else:
                roidb[i]['need_crop'] = 0
        else:
            roidb[i]['need_crop'] = 0

        ratio_list.append(ratio)

    ratio_list = np.array(ratio_list)
    ratio_index = np.argsort(ratio_list)
    return ratio_list[ratio_index], ratio_index 
Example 52
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: cocoeval.py    MIT License 5 votes vote down vote up
def computeIoU(self, imgId, catId):
        p = self.params
        if p.useCats:
            gt = self._gts[imgId,catId]
            dt = self._dts[imgId,catId]
        else:
            gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
            dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
        if len(gt) == 0 and len(dt) ==0:
            return []
        inds = np.argsort([-d['score'] for d in dt], kind='mergesort')
        dt = [dt[i] for i in inds]
        if len(dt) > p.maxDets[-1]:
            dt=dt[0:p.maxDets[-1]]

        if p.iouType == 'segm':
            g = [g['segmentation'] for g in gt]
            d = [d['segmentation'] for d in dt]
        elif p.iouType == 'bbox':
            g = [g['bbox'] for g in gt]
            d = [d['bbox'] for d in dt]
        else:
            raise Exception('unknown iouType for iou computation')

        # compute iou between each dt and gt region
        iscrowd = [int(o['iscrowd']) for o in gt]
        ious = maskUtils.iou(d,g,iscrowd)
        return ious 
Example 53
Project: Parallel.GAMIT   Author: demiangomez   File: test_voronoi.py    GNU General Public License v3.0 5 votes vote down vote up
def backbone_network(vstations, points, stns, ties):

    if isinstance(ties, list):
        # make a numpy array with the points if list passed
        ties = np.array(ties)

    flt = np.ones(len(ties), dtype=np.bool)

    # get xyz of the stations
    pc = ll2sphere_xyz(ties)
    print pc
    while len(pc[flt]) - BACKBONE_NET > 0:
        # calculate the spherical voronoi
        sv = SphericalVoronoi(pc[flt], radius=6371000, threshold=1e-9)
        #vor = Voronoi(lla)

        #fig = voronoi_plot_2d(vor, show_vertices=False, line_colors='orange',
        #                      line_width = 2, line_alpha = 0.6, point_size = 2)
        #plt.show()

        sv.sort_vertices_of_regions()

        area = np.zeros(len(sv.regions))
        weight = np.zeros(len(sv.regions))
        for i, region in enumerate(sv.regions):
            area[i] = calculate_surface_area_of_a_spherical_Voronoi_polygon(sv.vertices[region], 6371)
            weight[i] = 0.3 * area[i]  # also needs the stations weight (to do)

        # rank stations by weight
        minw = np.argsort(weight)
        #for m in minw:
        #    print stns[m][0], area[m]

        # remove the first one on the list
        flt[np.where(flt)[0][minw[0]]] = False

        print 'iterating %i' % len(pc[flt])

    plot_v(pc[flt], sv)
    return flt 
Example 54
Project: Parallel.GAMIT   Author: demiangomez   File: test_voronoi.py    GNU General Public License v3.0 5 votes vote down vote up
def smallestN_indices(a, N):
    """
    Function to return the row and column of the N smallest values
    :param a: array to search (any dimension)
    :param N: number of values to search
    :return: array with the rows-cols of min values
    """
    idx = a.ravel().argsort()[:N]
    return np.stack(np.unravel_index(idx, a.shape)).T 
Example 55
Project: transferlearning   Author: jindongwang   File: SCL.py    MIT License 5 votes vote down vote up
def fit(self, Xs, Xt):
        '''
        find pivot features and transfer the Xs and Xt
        Param Xs: source data
        Param Xt: target data
        output Xs_new: new source data features
        output Xt_new: new target data features
        output W: transform matrix
        '''
        _, ds = Xs.shape
        _, dt = Xt.shape
        assert ds == dt
        X = np.concatenate((Xs, Xt), axis=0)
        ix = np.argsort(np.sum(X, axis=0))
        ix = ix[::-1][:self.num_pivots]
        pivots = (X[:, ix]>0).astype('float')
        p = np.zeros((ds, self.num_pivots))
        # train for the classifers 
        for i in range(self.num_pivots):
            clf = linear_model.SGDClassifier(loss="modified_huber", alpha=self.l2)
            clf.fit(X, pivots[:, i])
            p[:, i] = clf.coef_
        _, W = np.linalg.eig(np.cov(p))
        W = W[:, :self.num_pivots].astype('float')
        self.W = W
        Xs_new = np.concatenate((np.dot(Xs, W), Xs), axis=1)
        Xt_new = np.concatenate((np.dot(Xt, W), Xt), axis=1)

        return Xs_new, Xt_new, W 
Example 56
Project: LorentzianDistanceRetrieval   Author: MarcTLaw   File: lorentzian_embed_with_normalized_rank.py    MIT License 5 votes vote down vote up
def ranking(types, model, distfn, data, order_rank = None):
    lt = th.from_numpy(model.embedding())
    embedding = Variable(lt, volatile=True)
    ranks = []
    ap_scores = []
    norms = []
    ordered_ranks = []
    for s, s_types in types.items():
        if order_rank is not None:
            lts = lt[s]            
            ltsnorm = th.sum(lts * lts, dim=-1)
            
            norms.append(float(ltsnorm[0]))
            ordered_ranks.append(order_rank[data.objects[s]])
            
        s_e = Variable(lt[s].expand_as(embedding), volatile=True)
        _dists = model.dist()(s_e, embedding).data.cpu().numpy().flatten()
        _dists[s] = 1e+12
        _labels = np.zeros(embedding.size(0))
        _dists_masked = _dists.copy()
        _ranks = []
        for o in s_types:
            _dists_masked[o] = np.Inf
            _labels[o] = 1
        ap_scores.append(average_precision_score(_labels, -_dists))
        for o in s_types:
            d = _dists_masked.copy()
            d[o] = _dists[o]
            r = np.argsort(d)
            _ranks.append(np.where(r == o)[0][0] + 1)
        ranks += _ranks
    rho = None
    if order_rank is not None:
        rho, pval = spearmanr(ordered_ranks,norms)

    return np.mean(ranks), np.mean(ap_scores), rho 
Example 57
Project: Deformable-ConvNets   Author: guanfuchen   File: box_annotator_ohem.py    MIT License 5 votes vote down vote up
def forward(self, is_train, req, in_data, out_data, aux):

        cls_score    = in_data[0]
        bbox_pred    = in_data[1]
        labels       = in_data[2].asnumpy()
        bbox_targets = in_data[3]
        bbox_weights = in_data[4]

        per_roi_loss_cls = mx.nd.SoftmaxActivation(cls_score) + 1e-14
        per_roi_loss_cls = per_roi_loss_cls.asnumpy()
        per_roi_loss_cls = per_roi_loss_cls[np.arange(per_roi_loss_cls.shape[0], dtype='int'), labels.astype('int')]
        per_roi_loss_cls = -1 * np.log(per_roi_loss_cls)
        per_roi_loss_cls = np.reshape(per_roi_loss_cls, newshape=(-1,))

        per_roi_loss_bbox = bbox_weights * mx.nd.smooth_l1((bbox_pred - bbox_targets), scalar=1.0)
        per_roi_loss_bbox = mx.nd.sum(per_roi_loss_bbox, axis=1).asnumpy()

        top_k_per_roi_loss = np.argsort(per_roi_loss_cls + per_roi_loss_bbox)
        labels_ohem = labels
        labels_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = -1
        bbox_weights_ohem = bbox_weights.asnumpy()
        bbox_weights_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = 0

        labels_ohem = mx.nd.array(labels_ohem)
        bbox_weights_ohem = mx.nd.array(bbox_weights_ohem)

        for ind, val in enumerate([labels_ohem, bbox_weights_ohem]):
            self.assign(out_data[ind], req[ind], val) 
Example 58
Project: Deformable-ConvNets   Author: guanfuchen   File: box_annotator_ohem.py    MIT License 5 votes vote down vote up
def forward(self, is_train, req, in_data, out_data, aux):

        cls_score    = in_data[0]
        bbox_pred    = in_data[1]
        labels       = in_data[2].asnumpy()
        bbox_targets = in_data[3]
        bbox_weights = in_data[4]

        per_roi_loss_cls = mx.nd.SoftmaxActivation(cls_score) + 1e-14
        per_roi_loss_cls = per_roi_loss_cls.asnumpy()
        per_roi_loss_cls = per_roi_loss_cls[np.arange(per_roi_loss_cls.shape[0], dtype='int'), labels.astype('int')]
        per_roi_loss_cls = -1 * np.log(per_roi_loss_cls)
        per_roi_loss_cls = np.reshape(per_roi_loss_cls, newshape=(-1,))

        per_roi_loss_bbox = bbox_weights * mx.nd.smooth_l1((bbox_pred - bbox_targets), scalar=1.0)
        per_roi_loss_bbox = mx.nd.sum(per_roi_loss_bbox, axis=1).asnumpy()

        top_k_per_roi_loss = np.argsort(per_roi_loss_cls + per_roi_loss_bbox)
        labels_ohem = labels
        labels_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = -1
        bbox_weights_ohem = bbox_weights.asnumpy()
        bbox_weights_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = 0

        labels_ohem = mx.nd.array(labels_ohem)
        bbox_weights_ohem = mx.nd.array(bbox_weights_ohem)

        for ind, val in enumerate([labels_ohem, bbox_weights_ohem]):
            self.assign(out_data[ind], req[ind], val) 
Example 59
Project: Deformable-ConvNets   Author: guanfuchen   File: box_annotator_ohem.py    MIT License 5 votes vote down vote up
def forward(self, is_train, req, in_data, out_data, aux):

        cls_score    = in_data[0]
        bbox_pred    = in_data[1]
        labels       = in_data[2].asnumpy()
        bbox_targets = in_data[3]
        bbox_weights = in_data[4]

        per_roi_loss_cls = mx.nd.SoftmaxActivation(cls_score) + 1e-14
        per_roi_loss_cls = per_roi_loss_cls.asnumpy()
        per_roi_loss_cls = per_roi_loss_cls[np.arange(per_roi_loss_cls.shape[0], dtype='int'), labels.astype('int')]
        per_roi_loss_cls = -1 * np.log(per_roi_loss_cls)
        per_roi_loss_cls = np.reshape(per_roi_loss_cls, newshape=(-1,))

        per_roi_loss_bbox = bbox_weights * mx.nd.smooth_l1((bbox_pred - bbox_targets), scalar=1.0)
        per_roi_loss_bbox = mx.nd.sum(per_roi_loss_bbox, axis=1).asnumpy()

        top_k_per_roi_loss = np.argsort(per_roi_loss_cls + per_roi_loss_bbox)
        labels_ohem = labels
        labels_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = -1
        bbox_weights_ohem = bbox_weights.asnumpy()
        bbox_weights_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = 0

        labels_ohem = mx.nd.array(labels_ohem)
        bbox_weights_ohem = mx.nd.array(bbox_weights_ohem)

        for ind, val in enumerate([labels_ohem, bbox_weights_ohem]):
            self.assign(out_data[ind], req[ind], val) 
Example 60
Project: Black-Box-Audio   Author: rtaori   File: run_audio_attack.py    MIT License 4 votes vote down vote up
def run(self, log=None):
        max_fitness_score = float('-inf') 
        dist = float('inf')
        best_text = ''
        itr = 1
        prev_loss = None
        if log is not None:
            log.write('target phrase: ' + self.target_phrase + '\n')
            log.write('itr, corr, lev dist \n')
        
        while itr <= self.max_iters and best_text != self.target_phrase:
            pop_scores, ctc = self.get_fitness_score(self.pop, self.target_phrase, self.input_audio)
            elite_ind = np.argsort(pop_scores)[-self.elite_size:]
            elite_pop, elite_pop_scores, elite_ctc = self.pop[elite_ind], pop_scores[elite_ind], ctc[elite_ind]
            
            if prev_loss is not None and prev_loss != elite_ctc[-1]: 
                self.mutation_p = self.mu * self.mutation_p + self.alpha / np.abs(prev_loss - elite_ctc[-1]) 

            if itr % 10 == 0:
                print('**************************** ITERATION {} ****************************'.format(itr))
                print('Current loss: {}'.format(-elite_ctc[-1]))
                save_wav(elite_pop[-1], self.output_wave_file)
                best_pop = np.tile(np.expand_dims(elite_pop[-1], axis=0), (100, 1))
                _, best_text = self.get_fitness_score(best_pop, self.target_phrase, self.input_audio, classify=True)
                
                dist = levenshteinDistance(best_text, self.target_phrase)
                corr = "{0:.4f}".format(np.corrcoef([self.input_audio, elite_pop[-1]])[0][1])
                print('Audio similarity to input: {}'.format(corr))
                print('Edit distance to target: {}'.format(dist))
                print('Currently decoded as: {}'.format(best_text))
                if log is not None:
                    log.write(str(itr) + ", " + corr + ", " + str(dist) + "\n")
                    
            if dist > 2:
                next_pop = get_new_pop(elite_pop, elite_pop_scores, self.pop_size)
                self.pop = mutate_pop(next_pop, self.mutation_p, self.noise_stdev, elite_pop)
                prev_loss = elite_ctc[-1]
                
            else:
                perturbed = np.tile(np.expand_dims(elite_pop[-1], axis=0), (self.num_points_estimate, 1))
                indices = np.random.choice(self.pop.shape[1], size=self.num_points_estimate, replace=False)

                perturbed[np.arange(self.num_points_estimate), indices] += self.delta_for_gradient
                perturbed_scores = self.get_fitness_score(perturbed, self.target_phrase, self.input_audio)[0]

                grad = (perturbed_scores - elite_ctc[-1]) / self.delta_for_gradient
                grad /= np.abs(grad).max()
                modified = elite_pop[-1].copy()
                modified[indices] += grad * self.delta_for_perturbation

                self.pop = np.tile(np.expand_dims(modified, axis=0), (self.pop_size, 1))
                self.delta_for_perturbation *= 0.995
                
            itr += 1

        return itr > self.max_iterations 
Example 61
Project: FRIDA   Author: LCAV   File: bands_selection.py    MIT License 4 votes vote down vote up
def select_bands(samples, freq_range, fs, nfft, win, n_bands, div=1):
    '''
    Selects the bins with most energy in a frequency range.

    It is possible to specify a div factor. Then the range is subdivided
    into div equal subbands and n_bands / div per subband are selected.
    '''

    if win is not None and isinstance(win, bool):
        if win:
            win = np.hanning(nfft)
        else:
            win = None

    # Read the signals in a single array
    sig = [wavfile.read(s)[1] for s in samples]
    L = max([s.shape[0] for s in sig])
    signals = np.zeros((L,len(samples)), dtype=np.float32)
    for i in range(signals.shape[1]):
        signals[:sig[i].shape[0],i] = sig[i] / np.std(sig[i][sig[i] > 1e-2])

    sum_sig = np.sum(signals, axis=1)

    sum_STFT = pra.stft(sum_sig, nfft, nfft, win=win, transform=rfft).T
    sum_STFT_avg = np.mean(np.abs(sum_STFT)**2, axis=1)

    # Do some band selection
    bnds = np.linspace(freq_range[0], freq_range[1], div+1)

    freq_hz = np.zeros(n_bands)
    freq_bins = np.zeros(n_bands, dtype=int)

    nsb = n_bands // div

    for i in range(div):

        bl = int(bnds[i] / fs * nfft)
        bh = int(bnds[i+1] / fs * nfft)

        k = np.argsort(sum_STFT_avg[bl:bh])[-nsb:]

        freq_hz[nsb*i:nsb*(i+1)] = (bl + k) / nfft * fs
        freq_bins[nsb*i:nsb*(i+1)] = k + bl

    freq_hz = freq_hz[:n_bands]

    return np.unique(freq_hz), np.unique(freq_bins) 
Example 62
Project: FRIDA   Author: LCAV   File: utils.py    MIT License 4 votes vote down vote up
def polar_distance(x1, x2):
    """
    Given two arrays of numbers x1 and x2, pairs the cells that are the
    closest and provides the pairing matrix index: x1(index(1,:)) should be as
    close as possible to x2(index(2,:)). The function outputs the average of the
    absolute value of the differences abs(x1(index(1,:))-x2(index(2,:))).
    :param x1: vector 1
    :param x2: vector 2
    :return: d: minimum distance between d
             index: the permutation matrix
    """
    x1 = np.reshape(x1, (1, -1), order='F')
    x2 = np.reshape(x2, (1, -1), order='F')
    N1 = x1.size
    N2 = x2.size
    diffmat = np.arccos(np.cos(x1 - np.reshape(x2, (-1, 1), order='F')))
    min_N1_N2 = np.min([N1, N2])
    index = np.zeros((min_N1_N2, 2), dtype=int)
    if min_N1_N2 > 1:
        for k in xrange(min_N1_N2):
            d2 = np.min(diffmat, axis=0)
            index2 = np.argmin(diffmat, axis=0)
            index1 = np.argmin(d2)
            index2 = index2[index1]
            index[k, :] = [index1, index2]
            diffmat[index2, :] = float('inf')
            diffmat[:, index1] = float('inf')
        d = np.mean(np.arccos(np.cos(x1[:, index[:, 0]] - x2[:, index[:, 1]])))
    else:
        d = np.min(diffmat)
        index = np.argmin(diffmat)
        if N1 == 1:
            index = np.array([1, index])
        else:
            index = np.array([index, 1])

    # sort to keep the order of the first vector
    if min_N1_N2 > 1:
        perm = np.argsort(index[:,0])
        index = index[perm,:]

    return d, index 
Example 63
Project: VSE-C   Author: ExplorerFreda   File: evaluation.py    MIT License 4 votes vote down vote up
def i2t(images, captions, npts=None, measure='cosine', return_ranks=False):
    """
    Images->Text (Image Annotation)
    Images: (5N, K) matrix of images
    Captions: (5N, K) matrix of captions
    """
    if npts is None:
        npts = images.shape[0] // 5
    index_list = []

    ranks = numpy.zeros(npts)
    top1 = numpy.zeros(npts)
    for index in range(npts):

        # Get query image
        im = images[5 * index].reshape(1, images.shape[1])

        # Compute scores
        if measure == 'order':
            bs = 100
            if index % bs == 0:
                mx = min(images.shape[0], 5 * (index + bs))
                im2 = images[5 * index:mx:5]
                d2 = order_sim(torch.Tensor(im2).cuda(),
                               torch.Tensor(captions).cuda())
                d2 = d2.cpu().numpy()
            d = d2[index % bs]
        else:
            d = numpy.dot(im, captions.T).flatten()
        inds = numpy.argsort(d)[::-1]
        index_list.append(inds[0])

        # Score
        rank = 1e20
        for i in range(5 * index, 5 * index + 5, 1):
            tmp = numpy.where(inds == i)[0][0]
            if tmp < rank:
                rank = tmp
        ranks[index] = rank
        top1[index] = inds[0]

    # Compute metrics
    r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
    r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
    r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
    medr = numpy.floor(numpy.median(ranks)) + 1
    meanr = ranks.mean() + 1
    if return_ranks:
        return (r1, r5, r10, medr, meanr), (ranks, top1)
    else:
        return r1, r5, r10, medr, meanr 
Example 64
Project: VSE-C   Author: ExplorerFreda   File: evaluation.py    MIT License 4 votes vote down vote up
def t2i(images, captions, npts=None, measure='cosine', return_ranks=False):
    """
    Text->Images (Image Search)
    Images: (5N, K) matrix of images
    Captions: (5N, K) matrix of captions
    """
    if npts is None:
        npts = images.shape[0] // 5
    ims = numpy.array([images[i] for i in range(0, len(images), 5)])
    ranks = numpy.zeros(5 * npts)
    top1 = numpy.zeros(5 * npts)
    for index in range(npts):

        # Get query captions
        queries = captions[5 * index:5 * index + 5]

        # Compute scores
        if measure == 'order':
            bs = 100
            if 5 * index % bs == 0:
                mx = min(captions.shape[0], 5 * index + bs)
                q2 = captions[5 * index:mx]
                d2 = order_sim(torch.Tensor(ims).cuda(),
                               torch.Tensor(q2).cuda())
                d2 = d2.cpu().numpy()

            d = d2[:, (5 * index) % bs:(5 * index) % bs + 5].T
        else:
            d = numpy.dot(queries, ims.T)
        inds = numpy.zeros(d.shape)
        for i in range(len(inds)):
            inds[i] = numpy.argsort(d[i])[::-1]
            ranks[5 * index + i] = numpy.where(inds[i] == index)[0][0]
            top1[5 * index + i] = inds[i][0]

    # Compute metrics
    r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
    r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
    r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
    medr = numpy.floor(numpy.median(ranks)) + 1
    meanr = ranks.mean() + 1
    if return_ranks:
        return (r1, r5, r10, medr, meanr), (ranks, top1)
    else:
        return r1, r5, r10, medr, meanr 
Example 65
Project: VSE-C   Author: ExplorerFreda   File: evaluation.py    MIT License 4 votes vote down vote up
def i2t_split(images, captions_orig, captions_ex, npts=None, measure='cosine', return_ranks=False):
    if npts is None:
        npts = images.shape[0] // 5
    index_list = []

    ranks = numpy.zeros(npts)
    top1 = numpy.zeros(npts)
    for index in range(npts):
        # Get query image
        im = images[5 * index].reshape(1, images.shape[1])
        captions_ex_ind = [captions_orig] + [captions_ex[5 * index + j] for j in range(5)]
        captions = np.concatenate(captions_ex_ind, axis=0)

        # Compute scores
        if measure == 'order':
            bs = 100
            if index % bs == 0:
                mx = min(images.shape[0], 5 * (index + bs))
                im2 = images[5 * index:mx:5]
                d2 = order_sim(torch.Tensor(im2).cuda(),
                               torch.Tensor(captions).cuda())
                d2 = d2.cpu().numpy()
            d = d2[index % bs]
        else:
            d = numpy.dot(im, captions.T).flatten()
#            print(captions.shape)
        inds = numpy.argsort(d)[::-1]
        index_list.append(inds[0])

        # Score
        rank = 1e20
        for i in range(5 * index, 5 * index + 5, 1):
            tmp = numpy.where(inds == i)[0][0]
            if tmp < rank:
                rank = tmp
        ranks[index] = rank
        top1[index] = inds[0]

    # Compute metrics
    r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
    r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
    r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
    medr = numpy.floor(numpy.median(ranks)) + 1
    meanr = ranks.mean() + 1
    if return_ranks:
        return (r1, r5, r10, medr, meanr), (ranks, top1)
    else:
        return r1, r5, r10, medr, meanr 
Example 66
Project: VSE-C   Author: ExplorerFreda   File: evaluation.py    MIT License 4 votes vote down vote up
def i2t_text_only(images, captions, npts=None, measure='cosine', return_ranks=False):
    """
    Images->Text (Image Annotation)
    Images: (5N, K) matrix of images
    Captions: (5N, K) matrix of captions
    """
    if npts is None:
        npts = images.shape[0] // 5
    index_list = []

    ranks = numpy.zeros(npts)
    top1 = numpy.zeros(npts)
    for index in range(npts):

        # Get query image
        im = images[5 * index].reshape(1, images.shape[1])

        # Compute scores
        if measure == 'order':
            bs = 100
            if index % bs == 0:
                mx = min(images.shape[0], 5 * (index + bs))
                im2 = images[5 * index:mx:5]
                d2 = order_sim(torch.Tensor(im2).cuda(),
                               torch.Tensor(captions).cuda())
                d2 = d2.cpu().numpy()
            d = d2[index % bs]
        else:
            d = numpy.dot(im, captions.T).flatten()
        pn_number = captions.shape[0] / images.shape[0]
        inds = numpy.argsort(d)[::-1]
        index_list.append(inds[0])

        # Score
        rank = 1e20
        for i in range(5 * index, 5 * index + 5, 1):
            tmp = numpy.where(inds == i * pn_number)[0][0]
            if tmp < rank:
                rank = tmp
        ranks[index] = rank
        top1[index] = inds[0]

    # Compute metrics
    r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
    r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
    r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
    medr = numpy.floor(numpy.median(ranks)) + 1
    meanr = ranks.mean() + 1
    if return_ranks:
        return (r1, r5, r10, medr, meanr), (ranks, top1)
    else:
        return r1, r5, r10, medr, meanr 
Example 67
Project: fuku-ml   Author: fukuball   File: DecisionStump.py    MIT License 4 votes vote down vote up
def train(self):

        if (self.status != 'init'):
            print("Please load train data and init W first.")
            return self.W

        self.status = 'train'

        error_in = self.data_num / self.data_num

        for i in range(0, self.train_X.shape[1]):

            dim_X = self.train_X[:, i]
            dim_XY = np.transpose(np.array([dim_X, self.train_Y]))
            sort_index = np.argsort(dim_XY[:, 0])
            sort_dim_XY = dim_XY[sort_index]
            sort_u = self.u[sort_index]

            sort_dim_X = sort_dim_XY[:, 0]
            sort_dim_Y = sort_dim_XY[:, 1]

            thetas = np.array([float("-inf")] + [(sort_dim_X[j] + sort_dim_X[j + 1]) / 2 for j in range(0, self.data_num - 1)] + [float("inf")])
            error_in_i = sum(sort_u)
            sign_i = 1
            theta_i = 0.0

            for theta in thetas:
                y_positive = np.where(sort_dim_X > theta, 1, -1)
                y_negative = np.where(sort_dim_X < theta, 1, -1)
                error_positive = sum((y_positive != sort_dim_Y) * sort_u)
                error_negative = sum((y_negative != sort_dim_Y) * sort_u)
                if error_positive > error_negative:
                    if error_in_i > error_negative:
                        error_in_i = error_negative
                        sign_i = -1
                        theta_i = theta
                else:
                    if error_in_i > error_positive:
                        error_in_i = error_positive
                        sign_i = 1
                        theta_i = theta

            if error_in > error_in_i:
                error_in = error_in_i
                self.sign = sign_i
                self.feature_index = i
                self.theta = theta_i

        return self.W 
Example 68
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: pascal_voc.py    Apache License 2.0 4 votes vote down vote up
def voc_eval(class_anno, npos, image_ids, bbox, confidence, ovthresh=0.5, use_07_metric=False):
        # sort by confidence
        if bbox.shape[0] > 0:
            sorted_inds = np.argsort(-confidence)
            sorted_scores = np.sort(-confidence)
            bbox = bbox[sorted_inds, :]
            image_ids = [image_ids[x] for x in sorted_inds]

        # go down detections and mark true positives and false positives
        nd = len(image_ids)
        tp = np.zeros(nd)
        fp = np.zeros(nd)
        for d in range(nd):
            r = class_anno[image_ids[d]]
            bb = bbox[d, :].astype(float)
            ovmax = -np.inf
            bbgt = r['bbox'].astype(float)

            if bbgt.size > 0:
                # compute overlaps
                # intersection
                ixmin = np.maximum(bbgt[:, 0], bb[0])
                iymin = np.maximum(bbgt[:, 1], bb[1])
                ixmax = np.minimum(bbgt[:, 2], bb[2])
                iymax = np.minimum(bbgt[:, 3], bb[3])
                iw = np.maximum(ixmax - ixmin + 1., 0.)
                ih = np.maximum(iymax - iymin + 1., 0.)
                inters = iw * ih

                # union
                uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                       (bbgt[:, 2] - bbgt[:, 0] + 1.) *
                       (bbgt[:, 3] - bbgt[:, 1] + 1.) - inters)

                overlaps = inters / uni
                ovmax = np.max(overlaps)
                jmax = np.argmax(overlaps)

            if ovmax > ovthresh:
                if not r['difficult'][jmax]:
                    if not r['det'][jmax]:
                        tp[d] = 1.
                        r['det'][jmax] = 1
                    else:
                        fp[d] = 1.
            else:
                fp[d] = 1.

        # compute precision recall
        fp = np.cumsum(fp)
        tp = np.cumsum(tp)
        rec = tp / float(npos)
        # avoid division by zero in case first detection matches a difficult ground ruth
        prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
        ap = PascalVOC.voc_ap(rec, prec, use_07_metric)

        return rec, prec, ap 
Example 69
Project: FCOS_GluonCV   Author: DetectionTeamUCAS   File: mask.py    Apache License 2.0 4 votes vote down vote up
def expand_mask(masks, bboxes, im_shape, scores=None, thresh=0.5):
    """Expand instance segmentation mask to full image size.

    Parameters
    ----------
    masks : numpy.ndarray or mxnet.nd.NDArray
        Binary images with shape `N, M, M`
    bboxes : numpy.ndarray or mxnet.nd.NDArray
        Bounding boxes with shape `N, 4`. Where `N` is the number of boxes
    im_shape : tuple
        Tuple of length 2: (width, height)
    scores : numpy.ndarray or mxnet.nd.NDArray, optional
        Confidence scores of the provided `bboxes` with shape `N`.
    thresh : float, optional, default 0.5
        Display threshold if `scores` is provided. Scores with less than `thresh`
        will be ignored in display, this is visually more elegant if you have
        a large number of bounding boxes with very small scores.

    Returns
    -------
    numpy.ndarray
        Binary images with shape `N, height, width`

    """
    if len(masks) != len(bboxes):
        raise ValueError('The length of bboxes and masks mismatch, {} vs {}'
                         .format(len(bboxes), len(masks)))
    if scores is not None and len(masks) != len(scores):
        raise ValueError('The length of scores and masks mismatch, {} vs {}'
                         .format(len(scores), len(masks)))

    if isinstance(masks, mx.nd.NDArray):
        masks = masks.asnumpy()
    if isinstance(bboxes, mx.nd.NDArray):
        bboxes = bboxes.asnumpy()
    if isinstance(scores, mx.nd.NDArray):
        scores = scores.asnumpy()

    areas = (bboxes[:, 2] - bboxes[:, 0]) * (bboxes[:, 3] - bboxes[:, 1])
    sorted_inds = np.argsort(-areas)

    full_masks = []
    for i in sorted_inds:
        if scores is not None and scores[i] < thresh:
            continue
        mask = masks[i]
        bbox = bboxes[i]
        full_masks.append(fill(mask, bbox, im_shape))
    full_masks = np.array(full_masks)
    return full_masks 
Example 70
Project: DOTA_models   Author: ringringyi   File: preprocess_dataset.py    Apache License 2.0 4 votes vote down vote up
def _build_vocabulary(input_files):
  """Loads or builds the model vocabulary.

  Args:
    input_files: List of pre-tokenized input .txt files.

  Returns:
    vocab: A dictionary of word to id.
  """
  if FLAGS.vocab_file:
    tf.logging.info("Loading existing vocab file.")
    vocab = collections.OrderedDict()
    with tf.gfile.GFile(FLAGS.vocab_file, mode="r") as f:
      for i, line in enumerate(f):
        word = line.decode("utf-8").strip()
        assert word not in vocab, "Attempting to add word twice: %s" % word
        vocab[word] = i
    tf.logging.info("Read vocab of size %d from %s",
                    len(vocab), FLAGS.vocab_file)
    return vocab

  tf.logging.info("Creating vocabulary.")
  num = 0
  wordcount = collections.Counter()
  for input_file in input_files:
    tf.logging.info("Processing file: %s", input_file)
    for sentence in tf.gfile.FastGFile(input_file):
      wordcount.update(sentence.split())

      num += 1
      if num % 1000000 == 0:
        tf.logging.info("Processed %d sentences", num)

  tf.logging.info("Processed %d sentences total", num)

  words = wordcount.keys()
  freqs = wordcount.values()
  sorted_indices = np.argsort(freqs)[::-1]

  vocab = collections.OrderedDict()
  vocab[special_words.EOS] = special_words.EOS_ID
  vocab[special_words.UNK] = special_words.UNK_ID
  for w_id, w_index in enumerate(sorted_indices[0:FLAGS.num_words - 2]):
    vocab[words[w_index]] = w_id + 2  # 0: EOS, 1: UNK.

  tf.logging.info("Created vocab with %d words", len(vocab))

  vocab_file = os.path.join(FLAGS.output_dir, "vocab.txt")
  with tf.gfile.FastGFile(vocab_file, "w") as f:
    f.write("\n".join(vocab.keys()))
  tf.logging.info("Wrote vocab file to %s", vocab_file)

  word_counts_file = os.path.join(FLAGS.output_dir, "word_counts.txt")
  with tf.gfile.FastGFile(word_counts_file, "w") as f:
    for i in sorted_indices:
      f.write("%s %d\n" % (words[i], freqs[i]))
  tf.logging.info("Wrote word counts file to %s", word_counts_file)

  return vocab 
Example 71
Project: DOTA_models   Author: ringringyi   File: metrics.py    Apache License 2.0 4 votes vote down vote up
def compute_precision_recall(scores, labels, num_gt):
  """Compute precision and recall.

  Args:
    scores: A float numpy array representing detection score
    labels: A boolean numpy array representing true/false positive labels
    num_gt: Number of ground truth instances

  Raises:
    ValueError: if the input is not of the correct format

  Returns:
    precision: Fraction of positive instances over detected ones. This value is
      None if no ground truth labels are present.
    recall: Fraction of detected positive instance over all positive instances.
      This value is None if no ground truth labels are present.

  """
  if not isinstance(
      labels, np.ndarray) or labels.dtype != np.bool or len(labels.shape) != 1:
    raise ValueError("labels must be single dimension bool numpy array")

  if not isinstance(
      scores, np.ndarray) or len(scores.shape) != 1:
    raise ValueError("scores must be single dimension numpy array")

  if num_gt < np.sum(labels):
    raise ValueError("Number of true positives must be smaller than num_gt.")

  if len(scores) != len(labels):
    raise ValueError("scores and labels must be of the same size.")

  if num_gt == 0:
    return None, None

  sorted_indices = np.argsort(scores)
  sorted_indices = sorted_indices[::-1]
  labels = labels.astype(int)
  true_positive_labels = labels[sorted_indices]
  false_positive_labels = 1 - true_positive_labels
  cum_true_positives = np.cumsum(true_positive_labels)
  cum_false_positives = np.cumsum(false_positive_labels)
  precision = cum_true_positives.astype(float) / (
      cum_true_positives + cum_false_positives)
  recall = cum_true_positives.astype(float) / num_gt
  return precision, recall 
Example 72
Project: pypriv   Author: soeaver   File: vis_mask_rcnn.py    MIT License 4 votes vote down vote up
def vis_one_image_opencv(im, boxes, segms=None, keypoints=None, thresh=0.9, kp_thresh=2,
                         show_box=False, show_kpts=False, show_mask=False, dataset=None, show_class=False):
    """Constructs a numpy array with the detections visualized."""

    if isinstance(boxes, list):
        boxes, segms, keypoints, classes = convert_from_cls_format(
            boxes, segms, keypoints)

    if boxes is None or boxes.shape[0] == 0 or max(boxes[:, 4]) < thresh:
        return im

    if segms is not None and len(segms) > 0:
        masks = mask_util.decode(segms)
        color_list = colormap()
        mask_color_id = 0

    # Display in largest to smallest order to reduce occlusion
    areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
    sorted_inds = np.argsort(-areas)

    for i in sorted_inds:
        bbox = boxes[i, :4]
        score = boxes[i, -1]
        if score < thresh:
            continue

        # show box (off by default)
        if show_box:
            im = vis_bbox(
                im, (bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]))

        # show class (off by default)
        if show_class:
            class_str = get_class_string(classes[i], score, dataset)
            im = vis_class(im, (bbox[0], bbox[1] - 2), class_str)

        # show mask
        if segms is not None and len(segms) > i and show_mask:
            color_mask = color_list[mask_color_id % len(color_list), 0:3]
            mask_color_id += 1
            im = vis_mask(im, masks[..., i], color_mask)

        # show keypoints
        if keypoints is not None and len(keypoints) > i and show_kpts:
            im = vis_keypoints(im, keypoints[i], kp_thresh)

    return im 
Example 73
Project: graph-neural-networks   Author: alelab-upenn   File: graphTools.py    GNU General Public License v3.0 4 votes vote down vote up
def permEDS(S):
    """
    permEDS: determines the permutation by the experimentally designed sampling
        score (from highest to lowest)

    Input:
        S (np.array): matrix

    Output:
        permS (np.array): matrix permuted
        order (list): list of indices to permute S to turn into permS.
    """
    # Design decisions: If we are given a multi-edge GSO, we're just going to
    # average all the edge dimensions and use that to compute the spectral
    # proxies.
    # Check S is of correct shape
    assert len(S.shape) == 2 or len(S.shape) == 3
    # If it is a matrix, just use it
    if len(S.shape) == 2:
        assert S.shape[0] == S.shape[1]
        scalarWeights = True
        simpleS = S.copy()
    # If it is a tensor of shape E x N x N, average over dimension E.
    else:
        assert S.shape[1] == S.shape[2]
        scalarWeights = False
        # Average over dimension E
        simpleS = np.mean(S, axis = 0)

    E, V = np.linalg.eig(simpleS) # Eigendecomposition of S
    kappa = np.max(np.absolute(V), axis=1)

    kappa2 = np.square(kappa) # The probabilities assigned to each node are
    # proportional to kappa2, so in the mean, the ones with largest kappa^2
    # would be "sampled" more often, and as suche are more important (i.e.
    # they have a higher score)

    # Sort ascending order (from min degree to max degree)
    order = np.argsort(kappa2)
    # Reverse sorting
    order = np.flip(order,0)

    if scalarWeights:
        S = S[order,:][:,order]
    else:
        S = S[:,order,:][:,:,order]

    return S, order.tolist() 
Example 74
Project: gullikson-scripts   Author: kgullikson88   File: Correlate.py    MIT License 4 votes vote down vote up
def get_rv(vel, corr, Npix=None, **fit_kws):
    """
    Get the best radial velocity, with errors.
    This will only work if the ccf was made with the maximum likelihood method!
    Uses the formula given in Zucker (2003) MNRAS, 342, 4  for the rv error.

    Parameters:
    ===========
    - vel:   numpy.ndarray
             The velocities
    - corr:  numpy.ndarray
             The ccf values. Should be the same size as vel
    - Npix:  integer
             The number of pixels used in the CCF.

    Returns:
    ========
    -rv:     float
             The best radial velocity, in km/s
    -rv_err: float
             Uncertainty in the velocity, in km/s
    -ccf:    float
             The CCF power at the maximum velocity.
    """
    vel = np.atleast_1d(vel)
    corr = np.atleast_1d(corr)
    sorter = np.argsort(vel)
    fcn = spline(vel[sorter], corr[sorter])
    fcn_prime = fcn.derivative(1)
    fcn_2prime = fcn.derivative(2)

    guess = vel[np.argmax(corr)]

    def errfcn(v):
        ll = 1e6*fcn_prime(v)**2
        return ll

    out = minimize_scalar(errfcn, bounds=(guess-2, guess+2), method='bounded')
    rv = out.x
    if Npix is None:
        Npix = vel.size
    
    rv_var = -(Npix * fcn_2prime(rv) * (fcn(rv) / (1 - fcn(rv) ** 2))) ** (-1)
    return rv, np.sqrt(rv_var), fcn(rv) 
Example 75
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: cocoeval.py    MIT License 4 votes vote down vote up
def computeOks(self, imgId, catId):
        p = self.params
        # dimention here should be Nxm
        gts = self._gts[imgId, catId]
        dts = self._dts[imgId, catId]
        inds = np.argsort([-d['score'] for d in dts], kind='mergesort')
        dts = [dts[i] for i in inds]
        if len(dts) > p.maxDets[-1]:
            dts = dts[0:p.maxDets[-1]]
        # if len(gts) == 0 and len(dts) == 0:
        if len(gts) == 0 or len(dts) == 0:
            return []
        ious = np.zeros((len(dts), len(gts)))
        sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0
        vars = (sigmas * 2)**2
        k = len(sigmas)
        # compute oks between each detection and ground truth object
        for j, gt in enumerate(gts):
            # create bounds for ignore regions(double the gt bbox)
            g = np.array(gt['keypoints'])
            xg = g[0::3]; yg = g[1::3]; vg = g[2::3]
            k1 = np.count_nonzero(vg > 0)
            bb = gt['bbox']
            x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2
            y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2
            for i, dt in enumerate(dts):
                d = np.array(dt['keypoints'])
                xd = d[0::3]; yd = d[1::3]
                if k1>0:
                    # measure the per-keypoint distance if keypoints visible
                    dx = xd - xg
                    dy = yd - yg
                else:
                    # measure minimum distance to keypoints in (x0,y0) & (x1,y1)
                    z = np.zeros((k))
                    dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0)
                    dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0)
                e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2
                if k1 > 0:
                    e=e[vg > 0]
                ious[i, j] = np.sum(np.exp(-e)) / e.shape[0]
        return ious 
Example 76
Project: transferlearning   Author: jindongwang   File: JDA.py    MIT License 4 votes vote down vote up
def fit_predict(self, Xs, Ys, Xt, Yt):
        '''
        Transform and Predict using 1NN as JDA paper did
        :param Xs: ns * n_feature, source feature
        :param Ys: ns * 1, source label
        :param Xt: nt * n_feature, target feature
        :param Yt: nt * 1, target label
        :return: acc, y_pred, list_acc
        '''
        list_acc = []
        X = np.hstack((Xs.T, Xt.T))
        X /= np.linalg.norm(X, axis=0)
        m, n = X.shape
        ns, nt = len(Xs), len(Xt)
        e = np.vstack((1 / ns * np.ones((ns, 1)), -1 / nt * np.ones((nt, 1))))
        C = len(np.unique(Ys))
        H = np.eye(n) - 1 / n * np.ones((n, n))

        M = 0
        Y_tar_pseudo = None
        for t in range(self.T):
            N = 0
            M0 = e * e.T * C
            if Y_tar_pseudo is not None and len(Y_tar_pseudo) == nt:
                for c in range(1, C + 1):
                    e = np.zeros((n, 1))
                    tt = Ys == c
                    e[np.where(tt == True)] = 1 / len(Ys[np.where(Ys == c)])
                    yy = Y_tar_pseudo == c
                    ind = np.where(yy == True)
                    inds = [item + ns for item in ind]
                    e[tuple(inds)] = -1 / len(Y_tar_pseudo[np.where(Y_tar_pseudo == c)])
                    e[np.isinf(e)] = 0
                    N = N + np.dot(e, e.T)
            M = M0 + N
            M = M / np.linalg.norm(M, 'fro')
            K = kernel(self.kernel_type, X, None, gamma=self.gamma)
            n_eye = m if self.kernel_type == 'primal' else n
            a, b = np.linalg.multi_dot([K, M, K.T]) + self.lamb * np.eye(n_eye), np.linalg.multi_dot([K, H, K.T])
            w, V = scipy.linalg.eig(a, b)
            ind = np.argsort(w)
            A = V[:, ind[:self.dim]]
            Z = np.dot(A.T, K)
            Z /= np.linalg.norm(Z, axis=0)
            Xs_new, Xt_new = Z[:, :ns].T, Z[:, ns:].T

            clf = KNeighborsClassifier(n_neighbors=1)
            clf.fit(Xs_new, Ys.ravel())
            Y_tar_pseudo = clf.predict(Xt_new)
            acc = sklearn.metrics.accuracy_score(Yt, Y_tar_pseudo)
            list_acc.append(acc)
            print('JDA iteration [{}/{}]: Acc: {:.4f}'.format(t + 1, self.T, acc))
        return acc, Y_tar_pseudo, list_acc 
Example 77
Project: transferlearning   Author: jindongwang   File: SFA.py    MIT License 4 votes vote down vote up
def fit(self, Xs, Xt):

        ix_s = np.argsort(np.sum(Xs, axis=0))
        ix_t = np.argsort(np.sum(Xt, axis=0))

        ix_s = ix_s[::-1][:self.l]
        ix_t = ix_t[::-1][:self.l]
        ix = np.intersect1d(ix_s, ix_t)
        _ix = np.setdiff1d(range(Xs.shape[1]), ix)
        self.ix = ix
        self._ix = _ix
        self.m = len(_ix)
        self.l = len(ix)

        X = np.concatenate((Xs, Xt), axis=0)
        DI = (X[:, ix] > 0).astype('float')
        DS = (X[:, _ix] > 0).astype('float')

        # construct co-occurrence matrix DSxDI
        M = np.zeros((self.m, self.l))
        for i in range(X.shape[0]):
            tem1 = np.reshape(DS[i], (1, self.m))
            tem2 = np.reshape(DI[i], (1, self.l))
            M += np.matmul(tem1.T, tem2)
        M = M/np.linalg.norm(M, 'fro')
        # #construct A matrix
        # tem_1 = np.zeros((self.m, self.m))
        # tem_2 = np.zeros((self.l, self.l))
        # A1 = np.concatenate((tem_1, M.T), axis=0)
        # A2 = np.concatenate((M, tem_2), axis=0)
        # A = np.concatenate((A1, A2), axis=1)
        # # compute laplace
        # D = np.zeros((A.shape[0], A.shape[1]))
        # for i in range(self.l+self.m):
        # 	D[i,i] = 1.0/np.sqrt(np.sum(A[i,:]))
        # L = (D.dot(A)).dot(D)
        # ut, _, _ = np.linalg.svd(L)
        M = sp.lil_matrix(M)
        D1 = sp.lil_matrix((self.m, self.m))
        D2 = sp.lil_matrix((self.l, self.l))
        for i in range(self.m):
            D1[i, i] = 1.0/np.sqrt(np.sum(M[1, :]).data[0])
        for i in range(self.l):
            D2[i, i] = 1.0/np.sqrt(np.sum(M[:, i]).T.data[0])
        B = (D1.tocsr().dot(M.tocsr())).dot(D2.tocsr())
        # print("Done.")
        # print("Computing SVD...")
        ut, s, vt = SVD(B.tocsc(), k=self.K)
        self.ut = ut
        return ut 
Example 78
Project: euclid   Author: njpayne   File: main.py    GNU General Public License v2.0 4 votes vote down vote up
def feature_reduction(header, data, is_classification = True):

    #remove low variance features
    data_lvar_removed, header_lvar_removed = clustering.clean_features(data, header, min_feature_variance = .8 * (1 - .8))

    #create training/test sets
    X_train, X_test, y_train, y_test = divide_for_training(data_lvar_removed)

    #select the best features using univatiate selection
    selected_features, feature_uni_scores = clustering.univariate_selection(np.vstack((X_train, X_test)), np.vstack((y_train, y_test)), n_best = 3, is_regression = not is_classification)

    #determine the order of the univariate features
    best_feature_index = np.argsort(-feature_uni_scores)

    best_results = []
  
    #reselect the features
    top_features = 12
    X_train_uni, X_test_uni = np.take(X_train, best_feature_index[ : top_features], axis = 1), np.take(X_test, best_feature_index[ : top_features], axis = 1)
    header_uni = np.take(header_lvar_removed, best_feature_index[ : top_features])

    ##try the PCA reduction
    #data_pca = clustering.pca_reduce(np.vstack((X_train, X_test)), n_components = top_features)
    #X_train_pca, X_test_pca, y_train_pca, y_test_pca = divide_for_training(np.hstack((np.vstack((y_train, y_test)), data_pca)))

    ##try the ICA reduction
    #data_ica = clustering.ica_reduce(np.vstack((X_train, X_test)), n_components = top_features)
    #X_train_ica, X_test_ica, y_train_ica, y_test_ica = divide_for_training(np.hstack((np.vstack((y_train, y_test)), data_ica)))

    #try recursive reduction
    data_recursive = clustering.recursive_reduce(np.vstack((X_train, X_test)), np.vstack((y_train, y_test)).flatten(), is_regression = not is_classification)
    X_train_recursive, X_test_recursive, y_train_recursive, y_test_recursive = divide_for_training(np.hstack((np.vstack((y_train, y_test)), data_ica)))

    if(is_classification):
        #run the classifiers and find the best result
        #best_classifier_uni = run_classifiers(X_train_uni, y_train, X_test_uni, y_test, header_uni)
        best_classifier_pca = run_classifiers(X_train_pca, y_train_pca, X_test_pca, y_test_pca, header_lvar_removed)
    else:
        #best_regressor_uni = run_regressors(X_train_uni, y_train, X_test_uni, y_test, header_uni)
        #best_regressor_pca = run_regressors(X_train_pca, y_train_pca, X_test_pca, y_test_pca, header_lvar_removed)
        #best_regressor_ica = run_regressors(X_train_ica, y_train_ica, X_test_ica, y_test_ica, header_lvar_removed)
        best_regressor_recursive = run_regressors(X_train_recursive, y_train_recursive, X_test_recursive, y_test_recursive, header_lvar_removed)
        best_results.append(max(best_regressor_ica, best_regressor_pca, best_regressor_uni))



    return 
Example 79
Project: SLiPy   Author: glentner   File: window_func.py    GNU General Public License v2.0 4 votes vote down vote up
def window_func2d(x,y,z,func,xmin=None,xmax=None,ymin=None,ymax=None,nbins=[10,10]):
	"""This function does compute a user-supplied function 
	on the subsets of y grouped by values of x
	E.g. imagine that you have x from 0 to 100 and you want
	to know the mean values of y for 0<x<10, 10<x<2- etc..
	In that case you need to do
	xbin,funcy,nperbin=window_func(x,y,lambda x: x.mean(),0,100,nbin=10)
	where xbin is the array with the centers of bins, 
	funcy is the func -- evaluated for y where x is within the appropriate bin
	and nperbin is the number of points in the appropriate bin
	empty keyword is needed if you want to retrieve the function evaluation 
	in empty bins too, otherwise they aren't returned at all
	"""
	
	if xmin is None:
		xmin = x.min()
	if xmax is None:
		xmax = x.max()
	if ymin is None:
		ymin = y.min()
	if ymax is None:
		ymax = y.max()

	hh, locx, locy = scipy.histogram2d(x, y,
			range=((xmin,xmax), (ymin,ymax)), bins=nbins)
	xinds = numpy.digitize(x, locx) - 1
	yinds = numpy.digitize(y, locy) - 1
	mask = numpy.zeros(hh.shape, bool)
	retv = hh  * 0.	
	subind = (xinds >= 0) & (yinds >= 0) & (xinds < nbins[0]) & (yinds < nbins[1])
	xinds, yinds, z1 = [_[subind] for _ in [xinds,yinds,z]]
	valind = yinds * (nbins[0]) + xinds
	sortind = np.argsort(valind)
	valind = valind[sortind]
	poss = np.where(np.diff(valind) > 0)[0]
	z1 = z1[sortind]
	for  i in range(len(poss) + 1):
		if i == 0:
			left = 0
			right = poss[0] + 1
		elif i == len(poss):
			left = poss[-1] + 1
			right = len(valind)
		else:
			left = poss[i-1] + 1
			right = poss[i] + 1
		curval = valind[left]
		retv[curval % (nbins[0]), curval / (nbins[0])] = func(z1[left:right])
	return retv, locx[0], locx[-1], locy[0], locy[-1], hh 
Example 80
Project: medicaldetectiontoolkit   Author: MIC-DKFZ   File: exp_utils.py    Apache License 2.0 4 votes vote down vote up
def run_model_selection(self, net, optimizer, monitor_metrics, epoch):

        # take the mean over all selection criteria in each epoch
        non_nan_scores = np.mean(np.array([[0 if ii is None else ii for ii in monitor_metrics['val'][sc]] for sc in self.cf.model_selection_criteria]), 0)
        epochs_scores = [ii for ii in non_nan_scores[1:]]
        # ranking of epochs according to model_selection_criterion
        epoch_ranking = np.argsort(epochs_scores)[::-1] + 1 #epochs start at 1
        # if set in configs, epochs < min_save_thresh are discarded from saving process.
        epoch_ranking = epoch_ranking[epoch_ranking >= self.cf.min_save_thresh]

        # check if current epoch is among the top-k epchs.
        if epoch in epoch_ranking[:self.cf.save_n_models]:

            save_dir = os.path.join(self.cf.fold_dir, '{}_best_checkpoint'.format(epoch))
            if not os.path.exists(save_dir):
                os.mkdir(save_dir)

            torch.save(net.state_dict(), os.path.join(save_dir, 'params.pth'))
            with open(os.path.join(save_dir, 'monitor_metrics.pickle'), 'wb') as handle:
                pickle.dump(monitor_metrics, handle)
            # save epoch_ranking to keep info for inference.
            np.save(os.path.join(self.cf.fold_dir, 'epoch_ranking'), epoch_ranking[:self.cf.save_n_models])
            np.save(os.path.join(save_dir, 'epoch_ranking'), epoch_ranking[:self.cf.save_n_models])

            self.logger.info(
                "saving current epoch {} at rank {}".format(epoch, np.argwhere(epoch_ranking == epoch)))
            # delete params of the epoch that just fell out of the top-k epochs.
            for se in [int(ii.split('_')[0]) for ii in os.listdir(self.cf.fold_dir) if 'best_checkpoint' in ii]:
                if se in epoch_ranking[self.cf.save_n_models:]:
                    subprocess.call('rm -rf {}'.format(os.path.join(self.cf.fold_dir, '{}_best_checkpoint'.format(se))), shell=True)
                    self.logger.info('deleting epoch {} at rank {}'.format(se, np.argwhere(epoch_ranking == se)))

        state = {
            'epoch': epoch,
            'state_dict': net.state_dict(),
            'optimizer': optimizer.state_dict(),
        }

        # save checkpoint of current epoch.
        save_dir = os.path.join(self.cf.fold_dir, 'last_checkpoint'.format(epoch))
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        torch.save(state, os.path.join(save_dir, 'params.pth'))
        np.save(os.path.join(save_dir, 'epoch_ranking'), epoch_ranking[:self.cf.save_n_models])
        with open(os.path.join(save_dir, 'monitor_metrics.pickle'), 'wb') as handle:
            pickle.dump(monitor_metrics, handle)