Python sklearn.metrics.roc_auc_score() Examples

The following are 30 code examples for showing how to use sklearn.metrics.roc_auc_score(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module sklearn.metrics , or try the search function .

Example 1
Project: edge2vec   Author: RoyZhengGao   File: multi_class_classification.py    License: BSD 3-Clause "New" or "Revised" License 7 votes vote down vote up
def multi_class_classification(data_X,data_Y):
    '''
    calculate multi-class classification and return related evaluation metrics
    '''

    svc = svm.SVC(C=1, kernel='linear')
    # X_train, X_test, y_train, y_test = train_test_split( data_X, data_Y, test_size=0.4, random_state=0) 
    clf = svc.fit(data_X, data_Y) #svm
    # array = svc.coef_
    # print array
    predicted = cross_val_predict(clf, data_X, data_Y, cv=2)
    print "accuracy",metrics.accuracy_score(data_Y, predicted)
    print "f1 score macro",metrics.f1_score(data_Y, predicted, average='macro') 
    print "f1 score micro",metrics.f1_score(data_Y, predicted, average='micro') 
    print "precision score",metrics.precision_score(data_Y, predicted, average='macro') 
    print "recall score",metrics.recall_score(data_Y, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(data_Y, predicted)
    print "classification_report", metrics.classification_report(data_Y, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(data_Y, predicted)
    # print "log_loss", metrics.log_loss(data_Y, predicted)
    print "zero_one_loss", metrics.zero_one_loss(data_Y, predicted)
    # print "AUC&ROC",metrics.roc_auc_score(data_Y, predicted)
    # print "matthews_corrcoef", metrics.matthews_corrcoef(data_Y, predicted) 
Example 2
Project: edge2vec   Author: RoyZhengGao   File: link_prediction.py    License: BSD 3-Clause "New" or "Revised" License 7 votes vote down vote up
def evaluation_analysis(true_label,predicted): 
    '''
    return all metrics results
    '''
    print "accuracy",metrics.accuracy_score(true_label, predicted)
    print "f1 score macro",metrics.f1_score(true_label, predicted, average='macro')     
    print "f1 score micro",metrics.f1_score(true_label, predicted, average='micro') 
    print "precision score",metrics.precision_score(true_label, predicted, average='macro') 
    print "recall score",metrics.recall_score(true_label, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(true_label, predicted)
    print "classification_report", metrics.classification_report(true_label, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(true_label, predicted)
    print "log_loss", metrics.log_loss(true_label, predicted)
    print "zero_one_loss", metrics.zero_one_loss(true_label, predicted)
    print "AUC&ROC",metrics.roc_auc_score(true_label, predicted)
    print "matthews_corrcoef", metrics.matthews_corrcoef(true_label, predicted) 
Example 3
Project: pytorch_geometric   Author: rusty1s   File: signed_gcn.py    License: MIT License 6 votes vote down vote up
def test(self, z, pos_edge_index, neg_edge_index):
        """Evaluates node embeddings :obj:`z` on positive and negative test
        edges by computing AUC and F1 scores.

        Args:
            z (Tensor): The node embeddings.
            pos_edge_index (LongTensor): The positive edge indices.
            neg_edge_index (LongTensor): The negative edge indices.
        """
        with torch.no_grad():
            pos_p = self.discriminate(z, pos_edge_index)[:, :2].max(dim=1)[1]
            neg_p = self.discriminate(z, neg_edge_index)[:, :2].max(dim=1)[1]
        pred = (1 - torch.cat([pos_p, neg_p])).cpu()
        y = torch.cat(
            [pred.new_ones((pos_p.size(0))),
             pred.new_zeros(neg_p.size(0))])
        pred, y = pred.numpy(), y.numpy()

        auc = roc_auc_score(y, pred)
        f1 = f1_score(y, pred, average='binary') if pred.sum() > 0 else 0

        return auc, f1 
Example 4
Project: pytorch_geometric   Author: rusty1s   File: autoencoder.py    License: MIT License 6 votes vote down vote up
def test(self, z, pos_edge_index, neg_edge_index):
        r"""Given latent variables :obj:`z`, positive edges
        :obj:`pos_edge_index` and negative edges :obj:`neg_edge_index`,
        computes area under the ROC curve (AUC) and average precision (AP)
        scores.

        Args:
            z (Tensor): The latent space :math:`\mathbf{Z}`.
            pos_edge_index (LongTensor): The positive edges to evaluate
                against.
            neg_edge_index (LongTensor): The negative edges to evaluate
                against.
        """
        pos_y = z.new_ones(pos_edge_index.size(1))
        neg_y = z.new_zeros(neg_edge_index.size(1))
        y = torch.cat([pos_y, neg_y], dim=0)

        pos_pred = self.decoder(z, pos_edge_index, sigmoid=True)
        neg_pred = self.decoder(z, neg_edge_index, sigmoid=True)
        pred = torch.cat([pos_pred, neg_pred], dim=0)

        y, pred = y.detach().cpu().numpy(), pred.detach().cpu().numpy()

        return roc_auc_score(y, pred), average_precision_score(y, pred) 
Example 5
Project: snape   Author: mbernico   File: score_dataset.py    License: Apache License 2.0 6 votes vote down vote up
def score_binary_classification(y, y_hat, report=True):
    """
    Create binary classification output
    :param y: true value
    :param y_hat: class 1 probabilities
    :param report:
    :return:
    """
    y_hat_class = [1 if x >= 0.5 else 0 for x in y_hat]  # convert probability to class for classification report

    report_string = "---Binary Classification Score--- \n"
    report_string += classification_report(y, y_hat_class)
    score = roc_auc_score(y, y_hat)
    report_string += "\nAUC = " + str(score)

    if report:
        print(report_string)

    return score, report_string 
Example 6
Project: metal   Author: HazyResearch   File: metrics.py    License: Apache License 2.0 6 votes vote down vote up
def roc_auc_score(gold, probs, ignore_in_gold=[], ignore_in_pred=[]):
    """Compute the ROC AUC score, given the gold labels and predicted probs.

    Args:
        gold: A 1d array-like of gold labels
        probs: A 2d array-like of predicted probabilities
        ignore_in_gold: A list of labels for which elements having that gold
            label will be ignored.

    Returns:
        roc_auc_score: The (float) roc_auc score
    """
    gold = arraylike_to_numpy(gold)

    # Filter out the ignore_in_gold (but not ignore_in_pred)
    # Note the current sub-functions (below) do not handle this...
    if len(ignore_in_pred) > 0:
        raise ValueError("ignore_in_pred not defined for ROC-AUC score.")
    keep = [x not in ignore_in_gold for x in gold]
    gold = gold[keep]
    probs = probs[keep, :]

    # Convert gold to one-hot indicator format, using the k inferred from probs
    gold_s = pred_to_prob(torch.from_numpy(gold), k=probs.shape[1]).numpy()
    return skm.roc_auc_score(gold_s, probs) 
Example 7
Project: geosketch   Author: brianhie   File: mouse_brain_subcluster.py    License: MIT License 6 votes vote down vote up
def auroc(X, genes, labels, focus, background=None):
    assert(len(genes) == X.shape[1])

    focus_idx = focus == labels
    if background is None:
        background_idx = range(X.shape[0])
    else:
        background_idx = background == labels
        
    for g, gene in enumerate(genes):
        x_gene = X[:, g]
        x_focus = x_gene[focus_idx]
        x_background = x_gene[background_idx]
        y_score = np.concatenate([ x_focus, x_background ])
        
        y_true = np.zeros(len(x_focus) + len(x_background))
        y_true[:len(x_focus)] = 1

        auroc = roc_auc_score(y_true, y_score)

        print('{}\t{}'.format(gene, auroc)) 
Example 8
Project: geosketch   Author: brianhie   File: mouse_brain_astrocyte.py    License: MIT License 6 votes vote down vote up
def auroc(X, genes, labels, focus, background=None):
    assert(len(genes) == X.shape[1])

    focus_idx = focus == labels
    if background is None:
        background_idx = range(X.shape[0])
    else:
        background_idx = background == labels
        
    for g, gene in enumerate(genes):
        x_gene = X[:, g]
        x_focus = x_gene[focus_idx]
        x_background = x_gene[background_idx]
        y_score = np.concatenate([ x_focus, x_background ])
        
        y_true = np.zeros(len(x_focus) + len(x_background))
        y_true[:len(x_focus)] = 1

        auroc = roc_auc_score(y_true, y_score)

        print('{}\t{}'.format(gene, auroc)) 
Example 9
Project: PADME   Author: simonfqy   File: __init__.py    License: MIT License 6 votes vote down vote up
def compute_roc_auc_scores(y, y_pred):
  """Transforms the results dict into roc-auc-scores and prints scores.

  Parameters
  ----------
  results: dict
  task_types: dict
    dict mapping task names to output type. Each output type must be either
    "classification" or "regression".
  """
  try:
    score = roc_auc_score(y, y_pred)
  except ValueError:
    warnings.warn("ROC AUC score calculation failed.")
    score = 0.5
  return score 
Example 10
Project: GCMC   Author: CrickWu   File: eval.py    License: MIT License 6 votes vote down vote up
def eval_all_scores(y_true_dict, y_score_dict):
    # when calculating RMSE, make sure y_true_dict is the full dict of list
    aps = []    # average precisions
    ndcgs = [[], [], []] # return ndcg at 1, 3, 5
    for q in y_true_dict:
        if q not in y_score_dict:
            raise ValueError("Prediction has missing items.")
        if np.sum(y_true_dict[q]) != 0:
            aps.append(average_precision(y_true_dict[q], y_score_dict[q]))
            ndcgs[0].append(ndcg_score(y_true_dict[q], y_score_dict[q], k=1))
            ndcgs[1].append(ndcg_score(y_true_dict[q], y_score_dict[q], k=3))
            ndcgs[2].append(ndcg_score(y_true_dict[q], y_score_dict[q], k=5))
    ndcgs = np.asarray(ndcgs)

    y_true_list = trans_dict_to_list(y_true_dict, y_true_dict)
    y_score_list = trans_dict_to_list(y_true_dict, y_score_dict)

    auc = roc_auc_score(y_true_list, y_score_list)
    rmse = np.mean((y_true_list - y_score_list)**2)

    # map, ndcg@1, ndcg@3, ndcg@5, auc, rmse
    return sum(aps)/len(aps), np.mean(ndcgs[0,:]), np.mean(ndcgs[1,:]), np.mean(ndcgs[2,:]), auc, np.sqrt(rmse)

# including MAP and AUC 
Example 11
Project: deep-mil-for-whole-mammogram-classification   Author: wentaozhu   File: roc_auc.py    License: MIT License 6 votes vote down vote up
def on_epoch_end(self, epoch, logs={}):
    if epoch % self.interval == 0:
      y_pred = self.model.predict(self.X_val, verbose=0)
      #print(np.sum(y_pred[:,1]))
      #y_true = np.argmax(self.y_val, axis=1)
      #y_pred = np.argmax(y_pred, axis=1)
      #print(y_true.shape, y_pred.shape)
      if self.mymil:
        score = roc_auc_score(self.y_val.max(axis=1), y_pred.max(axis=1))  
      else: score = roc_auc_score(self.y_val[:,1], y_pred[:,1])
      print("interval evaluation - epoch: {:d} - auc: {:.2f}".format(epoch, score))
      if score > self.auc:
        self.auc = score
        for f in os.listdir('./'):
          if f.startswith(self.filepath+'auc'):
            os.remove(f)
        self.model.save(self.filepath+'auc'+str(score)+'ep'+str(epoch)+'.hdf5') 
Example 12
Project: deep-mil-for-whole-mammogram-classification   Author: wentaozhu   File: roc_auc.py    License: MIT License 6 votes vote down vote up
def perform(self, node, inputs, output_storage):
        """
        Calculate ROC AUC score.

        Parameters
        ----------
        node : Apply instance
            Symbolic inputs and outputs.
        inputs : list
            Sequence of inputs.
        output_storage : list
            List of mutable 1-element lists.
        """
        if roc_auc_score is None:
            raise RuntimeError("Could not import from sklearn.")
        y_true, y_score = inputs
        try:
            roc_auc = roc_auc_score(y_true, y_score)
        except ValueError:
            roc_auc = np.nan
        #rvalue = np.array((roc_auc, prec, reca, f1))
        #[0][0]
        output_storage[0][0] = theano._asarray(roc_auc, dtype=config.floatX) 
Example 13
Project: inferbeddings   Author: uclnlp   File: metrics.py    License: MIT License 6 votes vote down vote up
def __call__(self, pos_triples, neg_triples=None):
        triples = pos_triples + neg_triples
        labels = [1 for _ in range(len(pos_triples))] + [0 for _ in range(len(neg_triples))]

        Xr, Xe = [], []
        for (s_idx, p_idx, o_idx), label in zip(triples, labels):
            Xr += [[p_idx]]
            Xe += [[s_idx, o_idx]]

        ascores = self.scoring_function([Xr, Xe])
        ays = np.array(labels)

        if self.rescale_predictions:
            diffs = np.diff(np.sort(ascores))
            min_diff = min(abs(diffs[np.nonzero(diffs)]))

            if min_diff < 1e-8:
                ascores = (ascores * (1e-7 / min_diff)).astype(np.float64)

        aucroc_value = metrics.roc_auc_score(ays, ascores)
        precision, recall, thresholds = metrics.precision_recall_curve(ays, ascores, pos_label=1)
        aucpr_value = metrics.auc(recall, precision)

        return aucroc_value, aucpr_value 
Example 14
Project: pyod   Author: yzhao062   File: data.py    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
def evaluate_print(clf_name, y, y_pred):
    """Utility function for evaluating and printing the results for examples.
    Default metrics include ROC and Precision @ n

    Parameters
    ----------
    clf_name : str
        The name of the detector.

    y : list or numpy array of shape (n_samples,)
        The ground truth. Binary (0: inliers, 1: outliers).

    y_pred : list or numpy array of shape (n_samples,)
        The raw outlier scores as returned by a fitted model.

    """

    y = column_or_1d(y)
    y_pred = column_or_1d(y_pred)
    check_consistent_length(y, y_pred)

    print('{clf_name} ROC:{roc}, precision @ rank n:{prn}'.format(
        clf_name=clf_name,
        roc=np.round(roc_auc_score(y, y_pred), decimals=4),
        prn=np.round(precision_n_scores(y, y_pred), decimals=4))) 
Example 15
Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 5 votes vote down vote up
def _evaluate_final(self, model, xy_test, batch_size, history):
        res = {}
        pred_test = None

        if 'val_acc' in history.history:
            res['val_acc'] = max(history.history['val_acc'])
            rev_ix = -1 - list(reversed(history.history['val_acc'])).index(res['val_acc'])
            res['val_loss'] = history.history['val_loss'][rev_ix]

        res['acc'] = history.history['acc'][-1]
        res['loss'] = history.history['loss'][-1]

        if len(xy_test[0]):
            from sklearn.metrics import classification_report, roc_auc_score
            # evaluate with test data
            x_test, y_test = xy_test
            pred_test = model.predict(x_test, batch_size=batch_size, verbose=0)
            test_loss, test_acc = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=0)
            res['test_loss'] = test_loss
            res['test_acc'] = test_acc

            report = classification_report(y_true = np.argmax(y_test, axis=1),
                                           y_pred = np.argmax(pred_test, axis=1),
                                           target_names=self.labels,
                                           digits=4,
                                           output_dict=True)

            res['auc'] = roc_auc_score(y_test.astype(np.int), pred_test)

            for label in self.labels:
                stats = report[label]
                res[label+"-precision"] = stats['precision']
                res[label+"-recall"] = stats['recall']
                res[label+"-f1"] = stats['f1-score']

        return pred_test, res 
Example 16
Project: EDeN   Author: fabriziocosta   File: estimator_utils.py    License: MIT License 5 votes vote down vote up
def perf(y_true, y_pred, y_score):
    """perf."""
    print('Accuracy: %.2f' % accuracy_score(y_true, y_pred))
    print(' AUC ROC: %.2f' % roc_auc_score(y_true, y_score))
    print('  AUC AP: %.2f' % average_precision_score(y_true, y_score))
    print()
    print('Classification Report:')
    print(classification_report(y_true, y_pred))
    print()
    plot_confusion_matrices(y_true, y_pred, size=int(len(set(y_true)) * 2.5))
    print()
    plot_aucs(y_true, y_score, size=10) 
Example 17
Project: EDeN   Author: fabriziocosta   File: ml.py    License: MIT License 5 votes vote down vote up
def estimate_model(positive_data_matrix=None,
                   negative_data_matrix=None,
                   target=None,
                   estimator=None,
                   n_jobs=4):
    """estimate_model."""
    X, y = make_data_matrix(positive_data_matrix=positive_data_matrix,
                            negative_data_matrix=negative_data_matrix,
                            target=target)
    logger.info('Test set')
    logger.info(describe(X))
    logger.info('-' * 80)
    logger.info('Test Estimate')
    predictions = estimator.predict(X)
    margins = estimator.decision_function(X)
    logger.info(classification_report(y, predictions))
    apr = average_precision_score(y, margins)
    logger.info('APR: %.3f' % apr)
    roc = roc_auc_score(y, margins)
    logger.info('ROC: %.3f' % roc)

    logger.info('Cross-validated estimate')
    scoring_strings = ['accuracy', 'precision', 'recall', 'f1',
                       'average_precision', 'roc_auc']
    for scoring in scoring_strings:
        scores = cross_validation.cross_val_score(
            estimator, X, y, cv=5,
            scoring=scoring, n_jobs=n_jobs)
        logger.info('%20s: %.3f +- %.3f' % (scoring,
                                            np.mean(scores),
                                            np.std(scores)))

    return roc, apr 
Example 18
Project: icme2019   Author: ShenDezhou   File: auc_util.py    License: MIT License 5 votes vote down vote up
def auroc(y_true, y_pred):
    return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double) 
Example 19
Project: Kaggler   Author: jeongyoonlee   File: categorical.py    License: MIT License 5 votes vote down vote up
def auc(y, p):
        return tf.py_function(roc_auc_score, (y, p), tf.double) 
Example 20
Project: 3DGCN   Author: blackmints   File: callback.py    License: MIT License 5 votes vote down vote up
def calculate_roc_pr(model, sequence, mask=-1, return_pred=False):
    y_true = sequence.y
    y_pred = model.predict_generator(sequence, use_multiprocessing=True, workers=6)

    if y_true.ndim == 1:
        val_roc = roc_auc_score(y_true, y_pred)
        val_pr = average_precision_score(y_true, y_pred)

    elif y_true.ndim == 2:
        y_true = y_true.transpose()
        y_pred = y_pred.transpose()

        unmask_idx = [np.where(y != mask)[0] for y in y_true]
        val_roc = [roc_auc_score(yt[idx], yp[idx]) for (yt, yp, idx) in zip(y_true, y_pred, unmask_idx)]
        val_pr = [average_precision_score(yt[idx], yp[idx]) for (yt, yp, idx) in zip(y_true, y_pred, unmask_idx)]

        val_roc = np.array(val_roc).mean()
        val_pr = np.array(val_pr).mean()
        y_pred = y_pred.transpose()

    else:
        raise ValueError("Unsupported output shape for auc calculation")

    if return_pred:
        return val_roc, val_pr, y_pred
    else:
        return val_roc, val_pr 
Example 21
Project: ocsvm-anomaly-detection   Author: hiram64   File: anomaly_detection_ocsvm.py    License: MIT License 5 votes vote down vote up
def calc_metrics(testy, scores):
    precision, recall, _ = precision_recall_curve(testy, scores)
    roc_auc = roc_auc_score(testy, scores)
    prc_auc = auc(recall, precision)

    return roc_auc, prc_auc 
Example 22
Project: fake-news-detection   Author: aldengolab   File: model.py    License: MIT License 5 votes vote down vote up
def auc_roc(self, y_true, y_scores):
        '''
        Computes the Area-Under-the-Curve for the ROC curve.
        '''
        return metrics.roc_auc_score(y_true, y_scores) 
Example 23
Project: pytorch-fm   Author: rixwew   File: main.py    License: MIT License 5 votes vote down vote up
def test(model, data_loader, device):
    model.eval()
    targets, predicts = list(), list()
    with torch.no_grad():
        for fields, target in tqdm.tqdm(data_loader, smoothing=0, mininterval=1.0):
            fields, target = fields.to(device), target.to(device)
            y = model(fields)
            targets.extend(target.tolist())
            predicts.extend(y.tolist())
    return roc_auc_score(targets, predicts) 
Example 24
Project: OpenChem   Author: Mariewelt   File: tox21_rnn_config.py    License: MIT License 5 votes vote down vote up
def multitask_auc(ground_truth, predicted):
    from sklearn.metrics import roc_auc_score
    import numpy as np
    import torch
    ground_truth = np.array(ground_truth)
    predicted = np.array(predicted)
    n_tasks = ground_truth.shape[1]
    auc = []
    for i in range(n_tasks):
        ind = np.where(ground_truth[:, i] != 999)[0]
        auc.append(roc_auc_score(ground_truth[ind, i], predicted[ind, i]))
    #if torch.distributed.get_rank() == 0:
    #    print(auc)
    return np.mean(auc) 
Example 25
Project: OpenChem   Author: Mariewelt   File: tox21_rnn_config.py    License: MIT License 5 votes vote down vote up
def multitask_auc(ground_truth, predicted):
    from sklearn.metrics import roc_auc_score
    import numpy as np
    import torch
    ground_truth = np.array(ground_truth)
    predicted = np.array(predicted)
    n_tasks = ground_truth.shape[1]
    auc = []
    for i in range(n_tasks):
        ind = np.where(ground_truth[:, i] != 999)[0]
        auc.append(roc_auc_score(ground_truth[ind, i], predicted[ind, i]))
    #if torch.distributed.get_rank() == 0:
    #    print(auc)
    return np.mean(auc) 
Example 26
Project: edge2vec   Author: RoyZhengGao   File: link_prediction.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def calculate_roc(y_true,y_pred):
    '''
    calculate AUROC score
    '''
    auroc = roc_auc_score(y_true, y_pred)
    return auroc 
Example 27
Project: MKLpy   Author: IvanoLauriola   File: model_selection.py    License: GNU General Public License v3.0 5 votes vote down vote up
def __def_score__(score):
    '''internal check'''
    score = score.lower()
    if score == 'roc_auc':
    	return roc_auc_score, 'decision_function'
    elif score == 'accuracy':
    	return accuracy_score, 'predict'
    elif score == 'f_score':
    	return f1_score, 'predict'
    else:
    	raise ValueError('%s is not a valid metric. Valid metrics are \'roc_auc\', \'accuracy\', or \'f_score\'.' % score) 
Example 28
Project: MKLpy   Author: IvanoLauriola   File: callbacks.py    License: GNU General Public License v3.0 5 votes vote down vote up
def on_step_end(self, step):
		if step % self.cooldown:
			return

		if self.metric == 'obj':
			current = self.model.solution.objective
		elif self.metric == 'val_obj':
			pass
		elif self.metric == 'auc':
			Kva = self.model.func_form(self.KLva, self.model.solution.weights)
			ys = self.model.learner.fit(self.model.solution.ker_matrix, self.model.Y) \
				.decision_function(Kva)
			current = roc_auc_score(self.Yva, ys)
		elif self.metric == 'accuracy':
			Kva = self.model.func_form(self.KLva, self.model.solution.weights)
			ys = self.model.learner.fit(self.model.solution.ker_matrix, self.model.Y) \
				.predict(Kva)
			current = accuracy_score(self.Yva, ys)
		else:
			raise Error('Metric error')

		self.vals.append(current)#ratio(self.model.solution.ker_matrix, self.model.Y))
		#print ('[%d][earlystopping] current' % step, \
		#	self.model.solution.objective, self.vals[-1], current)


		if self.monitor_op(current, self.best):
			self.best_solution = Solution(
				weights = self.model.solution.weights,
				objective = self.model.solution.objective,
				ker_matrix = self.model.solution.ker_matrix,
				)
			self.best = current
			self.wait = 0
		else:
			self.wait += 1
			if self.wait >= self.patience:
				self.stoppend_epoch = step
				self.model.convergence = True 
Example 29
Project: SGCN   Author: benedekrozemberczki   File: utils.py    License: GNU General Public License v3.0 5 votes vote down vote up
def calculate_auc(targets, predictions, edges):
    """
    Calculate performance measures on test dataset.
    :param targets: Target vector to predict.
    :param predictions: Predictions vector.
    :param edges: Edges dictionary with number of edges etc.
    :return auc: AUC value.
    :return f1: F1-score.
    """
    neg_ratio = len(edges["negative_edges"])/edges["ecount"]
    targets = [0 if target == 1 else 1 for target in targets]
    auc = roc_auc_score(targets, predictions)
    f1 = f1_score(targets, [1 if p > neg_ratio else 0 for p in  predictions])
    return auc, f1 
Example 30
Project: EvalNE   Author: Dru-Mara   File: score.py    License: MIT License 5 votes vote down vote up
def auroc(self):
        r"""
        Computes the area under the ROC curve score.

        Returns
        -------
        auroc : float
            A value indicating the area under the ROC curve score.
        """
        tolerance = 0.25
        if np.sum(self.y_true) < tolerance * len(self.y_true) or \
                np.sum(self.y_true) > (1 - tolerance) * len(self.y_true):
            warnings.warn('AUROC is not recommended in the case of extreme class imbalance. ', Warning)
        return roc_auc_score(self.y_true, self.y_pred)