我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用sklearn.metrics.roc_curve()。
def calc_auc(y_pred_proba, labels, exp_run_folder, classifier, fold): auc = roc_auc_score(labels, y_pred_proba) fpr, tpr, thresholds = roc_curve(labels, y_pred_proba) curve_roc = np.array([fpr, tpr]) dataile_id = open(exp_run_folder+'/data/roc_{}_{}.txt'.format(classifier, fold), 'w+') np.savetxt(dataile_id, curve_roc) dataile_id.close() plt.plot(fpr, tpr, label='ROC curve: AUC={0:0.2f}'.format(auc)) plt.xlabel('1-Specificity') plt.ylabel('Sensitivity') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.grid(True) plt.title('ROC Fold {}'.format(fold)) plt.legend(loc="lower left") plt.savefig(exp_run_folder+'/data/roc_{}_{}.pdf'.format(classifier, fold), format='pdf') return auc
def plot_ROC(test_labels, test_predictions): fpr, tpr, thresholds = metrics.roc_curve( test_labels, test_predictions, pos_label=1) auc = "%.2f" % metrics.auc(fpr, tpr) title = 'ROC Curve, AUC = '+str(auc) with plt.style.context(('ggplot')): fig, ax = plt.subplots() ax.plot(fpr, tpr, "#000099", label='ROC curve') ax.plot([0, 1], [0, 1], 'k--', label='Baseline') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.legend(loc='lower right') plt.title(title) return fig
def getAUC(self,test_tasks): mean_tpr = 0.0 mean_fpr = np.linspace(0, 1, 100) for t in range(self.n_tasks): X_t, Y_t = self.extractTaskData(self.train_tasks,t) X_test_t, Y_test_t = self.extractTaskData(test_tasks, t) overallKernel = self.constructKernelFunction(t) self.classifiers[t] = SVC(C=self.C, kernel=overallKernel, probability=True, max_iter=self.max_iter_internal, tol=self.tolerance) probas_ = self.classifiers[t].fit(X_t, Y_t).predict_proba(X_test_t) fpr, tpr, thresholds = roc_curve(Y_test_t, probas_[:, 1]) mean_tpr += interp(mean_fpr, fpr, tpr) mean_tpr[0] = 0.0 mean_tpr /= self.n_tasks mean_tpr[-1] = 1.0 mean_auc = auc(mean_fpr, mean_tpr) return mean_auc, mean_fpr, mean_tpr
def pred_accuracy(y_true, y_pred): y_true = sp.copy(y_true) if len(sp.unique(y_true))==2: print 'dichotomous trait, calculating AUC' y_min = y_true.min() y_max = y_true.max() if y_min!= 0 or y_max!=1: y_true[y_true==y_min]=0 y_true[y_true==y_max]=1 fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred) auc = metrics.auc(fpr, tpr) return auc else: print 'continuous trait, calculating COR' cor = sp.corrcoef(y_true,y_pred)[0,1] return cor
def computeFROC(FROCGTList, FROCProbList, totalNumberOfImages, excludeList): # Remove excluded candidates FROCGTList_local = [] FROCProbList_local = [] for i in range(len(excludeList)): if excludeList[i] == False: FROCGTList_local.append(FROCGTList[i]) FROCProbList_local.append(FROCProbList[i]) numberOfDetectedLesions = sum(FROCGTList_local) totalNumberOfLesions = sum(FROCGTList) totalNumberOfCandidates = len(FROCProbList_local) fpr, tpr, thresholds = skl_metrics.roc_curve(FROCGTList_local, FROCProbList_local) if sum(FROCGTList) == len(FROCGTList): # Handle border case when there are no false positives and ROC analysis give nan values. print "WARNING, this system has no false positives.." fps = np.zeros(len(fpr)) else: fps = fpr * (totalNumberOfCandidates - numberOfDetectedLesions) / totalNumberOfImages sens = (tpr * numberOfDetectedLesions) / totalNumberOfLesions return fps, sens, thresholds
def _update_tsg_metrics(self, y_true, y_pred, prob): self.tsg_gene_pred = pd.Series(y_pred, self.y.index) self.tsg_gene_score = pd.Series(prob, self.y.index) # compute metrics for classification self.tsg_gene_count[self.num_pred] = sum(y_pred) prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred) tsg_col = 1 # column for metrics relate to tsg self.tsg_precision[self.num_pred] = prec[tsg_col] self.tsg_recall[self.num_pred] = recall[tsg_col] self.tsg_f1_score[self.num_pred] = fscore[tsg_col] self.logger.debug('Tsg Iter %d: Precission=%s, Recall=%s, f1_score=%s' % ( self.num_pred + 1, str(prec), str(recall), str(fscore))) # compute ROC curve metrics fpr, tpr, thresholds = metrics.roc_curve(y_true, prob) self.tsg_tpr_array[self.num_pred, :] = interp(self.tsg_fpr_array, fpr, tpr) #self.tsg_tpr_array[0] = 0.0 # compute Precision-Recall curve metrics p, r, thresh = metrics.precision_recall_curve(y_true, prob) p, r, thresh = p[::-1], r[::-1], thresh[::-1] # reverse order of results self.tsg_precision_array[self.num_pred, :] = interp(self.tsg_recall_array, r, p)
def addFold(self, fold_id, true_labels, predicted_proba, predicted_scores): if len(true_labels) == 0: return if self.probabilist_model: scores = predicted_proba else: scores = predicted_scores fpr, tpr, thresholds = roc_curve(true_labels, scores) self.mean_tpr += interp(self.mean_fpr, fpr, tpr) self.thresholds = interp(self.mean_fpr, fpr, thresholds) self.mean_tpr[0] = 0.0 self.thresholds[0] = 1.0 self.thresholds[-1] = 0.0 roc_auc = auc(fpr, tpr) if self.num_folds > 1: self.ax1.plot(fpr, tpr, lw = 1, label = 'ROC fold %d (area = %0.2f)' % (fold_id, roc_auc)) else: self.ax1.plot(fpr, tpr, lw = 3, color = colors_tools.getLabelColor('all'), label = 'ROC (area = %0.2f)' % (roc_auc))
def plot_roc_curve(y_true, y_score, ax=None): ''' Plot the Receiving Operator Characteristic curved, including the Area under the Curve (AUC) score. Parameters ---------- y_true : array y_score : array ax : matplotlib.axes, defaults to new axes Returns ------- ax : matplotlib.axes ''' ax = ax or plt.axes() auc = metrics.roc_auc_score(y_true, y_score) fpr, tpr, _ = metrics.roc_curve(y_true, y_score) ax.plot(fpr, tpr) ax.annotate('AUC: {:.2f}'.format(auc), (.8, .2)) ax.plot([0, 1], [0, 1], linestyle='--', color='k') return ax
def get_auc(outputs, probas): ''' AUC is a common metric for binary classification methods by comparing true & false positive rates Args ---- outputs : numpy array true outcomes (OxTxN) probas : numpy array predicted probabilities (OxTxN) Returns ------- auc : integer ''' fpr, tpr, _ = roc_curve(outputs, probas[:, 1]) return auc(fpr, tpr)
def plot_roc_curve(true_y, prob_y, out_file=None): from sklearn.metrics import roc_curve fpr, tpr, _ = roc_curve(true_y, prob_y) fig = plt.figure() plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve') plt.plot([0, 1], [0, 1], color='navy', lw=1, linestyle='--') plt.xlim([-0.025, 1.025]) plt.ylim([-0.025, 1.025]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('RoC Curve') if out_file is not None: fig.savefig(out_file) return fig
def plot_auc(self, estimator, estimator_name, neg, pos): try: classifier_probas = estimator.decision_function(self.X_test) except AttributeError: classifier_probas = estimator.predict_proba(self.X_test)[:, 1] false_positive_r, true_positive_r, thresholds = metrics.roc_curve(self.y_test, classifier_probas) roc_auc = metrics.auc(false_positive_r, true_positive_r) label = '{:.1f}% neg:{} pos:{} {}'.format(roc_auc * 100, neg, pos, estimator_name) plt.plot(false_positive_r, true_positive_r, label=label) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([-0.05, 1.0]) plt.ylim([0.0, 1.05]) plt.title('ROC score(s)') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.legend(loc='lower right', prop={'size': 10}) plt.savefig("ROC.png", dpi=300, bbox_inches='tight') plt.grid()
def get_fpr_tpr_roc(model, test_data, test_truth, labels): y_pred = model.predict(test_data, batch_size=32, verbose=0) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for k in labels.keys(): cur_idx = labels[k] fpr[labels[k]], tpr[labels[k]], _ = roc_curve(test_truth[:,cur_idx], y_pred[:,cur_idx]) roc_auc[labels[k]] = auc(fpr[labels[k]], tpr[labels[k]]) # Compute micro-average ROC curve and ROC area fpr["micro"], tpr["micro"], _ = roc_curve(test_truth.ravel(), y_pred.ravel()) roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) return fpr, tpr, roc_auc
def test_all_metrics(model, data=None, usage_ratio=1): if data is None: X_train, y_train, X_test, y_test = read_data(usage_ratio=usage_ratio) else: # You ought to use the same training & testing set from your initial input. X_train, y_train, X_test, y_test = data y_pred = model.predict_classes(X_test) y_ground = np.argmax(y_test, axis=1) # y_proba = model.predict_proba(X_test) # overall_acc = (y_pred == y_ground).sum() * 1. / y_pred.shape[0] precision = sk.metrics.precision_score(y_ground, y_pred) recall = sk.metrics.recall_score(y_ground, y_pred) f1_score = sk.metrics.f1_score(y_ground, y_pred) # confusion_matrix = sk.metrics.confusion_matrix(y_ground, y_pred) # fpr, tpr, thresholds = sk.metrics.roc_curve(y_ground, y_pred) print "precision_score = ", precision print "recall_score = ", recall print "f1_score = ", f1_score # plot_roc_curve(y_test, y_proba) plot_confusion_matrix(y_ground, y_pred)
def plot_ROC_by_class(y_true, y_pred, classes, ls='-'): print y_true.shape print y_pred.shape best_thresh = {} for class_name, c in classes.items(): # for each class # Compute ROC curve fpr, tpr, thresholds = roc_curve(y_true[:, c], y_pred[:, c]) roc_auc = auc(fpr, tpr) # Plot ROC curve plt.plot(fpr, tpr, label='{}, AUC = {:.3f}'.format(class_name, roc_auc), linestyle=ls) # Calculate J statistic J = [j_statistic(y_true[:, c], y_pred[:, c], t) for t in thresholds] j_best = np.argmax(J) # Store best threshold for each class best_thresh[class_name] = J[j_best] return best_thresh
def plot_roc_auc(predictions, ground_truth, name=''): # Calculate ROC curve y_pred = np.asarray(predictions).ravel() y_true = np.asarray(ground_truth).ravel() fpr, tpr, thresholds = roc_curve(y_true, y_pred) roc_auc = auc(fpr, tpr) # Plot plt.plot(fpr, tpr, label='{}, AUC = {:.3f}'.format(name, roc_auc)) # # Return index of best model by J statistic # J = [j_statistic(y_true, y_pred, t) for t in thresholds] # # return thresholds[np.argmax(J)] # TODO test this out!
def print_roc(self, y_true, y_scores, filename): ''' Prints the ROC for this model. ''' fpr, tpr, thresholds = metrics.roc_curve(y_true, y_scores) plt.figure() plt.plot(fpr, tpr, color='darkorange', label='ROC curve (area = %0.2f)' % self.roc_auc) plt.plot([0, 1], [0, 1], color='navy', linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic') plt.legend(loc="lower right") plt.savefig(filename) plt.close()
def default_inv_roc_curve(Y_true, var, sample_weight=None): """Default ROC curve for a single variable. Args: Y_true: array of true classes (n*2). var: array of variable values. sample_weight: array of sample weights. Returns: Array of (signal efficiency, 1/[background efficiency]) pairs. """ fpr, tpr, _ = roc_curve(Y_true[:, 0], var, sample_weight=sample_weight) print("AUC: {0:.4f}".format(auc(fpr, tpr, reorder=True))) res = 1./len(Y_true) return np.array([[tp, 1./max(fp, res)] for tp,fp in zip(tpr,fpr) if fp > 0.])
def plot_roc(y_test, y_pred, label=''): """Compute ROC curve and ROC area""" fpr, tpr, _ = roc_curve(y_test, y_pred) roc_auc = auc(fpr, tpr) # Plot of a ROC curve for a specific class plt.figure() plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic' + label) plt.legend(loc="lower right") plt.show()
def compute_roc(probs_neg, probs_pos, plot=False): """ TODO :param probs_neg: :param probs_pos: :param plot: :return: """ probs = np.concatenate((probs_neg, probs_pos)) labels = np.concatenate((np.zeros_like(probs_neg), np.ones_like(probs_pos))) fpr, tpr, _ = roc_curve(labels, probs) auc_score = auc(fpr, tpr) if plot: plt.figure(figsize=(7, 6)) plt.plot(fpr, tpr, color='blue', label='ROC (AUC = %0.4f)' % auc_score) plt.legend(loc='lower right') plt.title("ROC Curve") plt.xlabel("FPR") plt.ylabel("TPR") plt.show() return fpr, tpr, auc_score
def plot_roc(self): for learner, clf in self._clf.iteritems(): # Make the predictions (X_test, y_test) = self._test_data y_pred = clf.predict(X_test) # Get (f)alse (p)ositive (r)ate, (t)rue (p)ositive (r)ate fpr, tpr, _ = roc_curve(y_test, y_pred) # Add this classifier's results to the plot plt.plot(fpr, tpr, label='%s (area = %0.2f)'\ % (learner, auc(fpr, tpr))) # Now do the plot # NOTE: plot code stolen from scikit-learn docs (http://bit.ly/236k6M3) plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver Operating Characteristic (ROC)') plt.legend(loc="lower right") plt.show()
def print_misclassified(y, pred, files, fom_func, threshold): #fpr, tpr, thresholds = roc_curve(y, pred) #fom = 0.01 #FoMs.append(1-tpr[np.where(fpr<=FPR)[0][-1]]) #FoM, threshold, fpr, tpr = fom_func(y, pred, fom) negatives = np.where(y==0) positives = np.where(y==1) falsePositives = files[negatives][np.where(pred[negatives]>threshold)] print "[+] False positives (%d):" % len(falsePositives) for i,falsePositive in enumerate(falsePositives): print "\t " + str(falsePositive), pred[negatives][np.where(pred[negatives]>threshold)][i] print missedDetections = files[positives][np.where(pred[positives]<=threshold)] print "[+] Missed Detections (%d):" % len(missedDetections) for i,missedDetection in enumerate(missedDetections): print "\t " + str(missedDetection), pred[positives][np.where(pred[positives]<=threshold)][i] print
def evaluate(self, data, labels, site, sess=None): """ Runs one evaluation against the full epoch of data. Return the precision and the number of correct predictions. Batch evaluation saves memory and enables this to run on smaller GPUs. sess: the session in which the model has been trained. op: the Tensor that returns the number of correct predictions. data: size N x M N: number of signals (samples) M: number of vertices (features) labels: size N N: number of signals (samples) """ t_process, t_wall = time.process_time(), time.time() scores, loss = self.predict(data, labels, site, sess) fpr, tpr, _ = roc_curve(labels, scores) roc_auc = auc(fpr, tpr) string = 'samples: {:d}, AUC : {:.2f}, loss: {:.4e}'.format(len(labels), roc_auc, loss) if sess is None: string += '\ntime: {:.0f}s (wall {:.0f}s)'.format(time.process_time() - t_process, time.time() - t_wall) return string, roc_auc, loss, scores
def clf_scores(clf, x_train, y_train, x_test, y_test): info = dict() # TODO: extend this to a confusion matrix per fold for more flexibility downstream (tuning) # TODO: calculate a set of ROC curves per fold instead of running it on test, currently introducing bias scores = cross_val_score(clf, x_train, y_train, cv=cv, n_jobs=-1) runtime = time() clf.fit(x_train, y_train) runtime = time() - runtime y_test_predicted = clf.predict(x_test) info['runtime'] = runtime info['accuracy'] = min(scores) info['accuracy_test'] = accuracy_score(y_test, y_test_predicted) info['accuracy_folds'] = scores info['confusion_matrix'] = confusion_matrix(y_test, y_test_predicted) clf.fit(x_train, y_train) fpr, tpr, _ = roc_curve(y_test, clf_predict_proba(clf, x_test)) info['fpr'] = fpr info['tpr'] = tpr info['auc'] = auc(fpr, tpr) return info
def test_roc_returns_consistency(): # Test whether the returned threshold matches up with tpr # make small toy dataset y_true, _, probas_pred = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, probas_pred) # use the given thresholds to determine the tpr tpr_correct = [] for t in thresholds: tp = np.sum((probas_pred >= t) & y_true) p = np.sum(y_true) tpr_correct.append(1.0 * tp / p) # compare tpr and tpr_correct to see if the thresholds' order was correct assert_array_almost_equal(tpr, tpr_correct, decimal=2) assert_equal(fpr.shape, tpr.shape) assert_equal(fpr.shape, thresholds.shape)
def test_roc_nonrepeating_thresholds(): # Test to ensure that we don't return spurious repeating thresholds. # Duplicated thresholds can arise due to machine precision issues. dataset = datasets.load_digits() X = dataset['data'] y = dataset['target'] # This random forest classifier can only return probabilities # significant to two decimal places clf = ensemble.RandomForestClassifier(n_estimators=100, random_state=0) # How well can the classifier predict whether a digit is less than 5? # This task contributes floating point roundoff errors to the probabilities train, test = slice(None, None, 2), slice(1, None, 2) probas_pred = clf.fit(X[train], y[train]).predict_proba(X[test]) y_score = probas_pred[:, :5].sum(axis=1) # roundoff errors begin here y_true = [yy < 5 for yy in y[test]] # Check for repeating values in the thresholds fpr, tpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=False) assert_equal(thresholds.size, np.unique(np.round(thresholds, 2)).size)
def test_roc_curve_one_label(): y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] y_pred = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] # assert there are warnings w = UndefinedMetricWarning fpr, tpr, thresholds = assert_warns(w, roc_curve, y_true, y_pred) # all true labels, all fpr should be nan assert_array_equal(fpr, np.nan * np.ones(len(thresholds))) assert_equal(fpr.shape, tpr.shape) assert_equal(fpr.shape, thresholds.shape) # assert there are warnings fpr, tpr, thresholds = assert_warns(w, roc_curve, [1 - x for x in y_true], y_pred) # all negative labels, all tpr should be nan assert_array_equal(tpr, np.nan * np.ones(len(thresholds))) assert_equal(fpr.shape, tpr.shape) assert_equal(fpr.shape, thresholds.shape)
def plot_ROC(actual, predictions): # plot the FPR vs TPR and AUC for a two class problem (0,1) import matplotlib.pyplot as plt from sklearn.metrics import roc_curve, auc false_positive_rate, true_positive_rate, thresholds = roc_curve(actual, predictions) roc_auc = auc(false_positive_rate, true_positive_rate) plt.title('Receiver Operating Characteristic') plt.plot(false_positive_rate, true_positive_rate, 'b', label='AUC = %0.2f'% roc_auc) plt.legend(loc='lower right') plt.plot([0,1],[0,1],'r--') plt.xlim([-0.1,1.2]) plt.ylim([-0.1,1.2]) plt.ylabel('True Positive Rate') plt.xlabel('False Positive Rate') plt.show()
def metric(model, test_csv, fname): X, Y_true, headers = get_XY(test_csv) Y_pred = model.predict(X) try: print confusion_matrix(Y_true, [a[0] > 0.5 for a in Y_pred]) except IndexError: print confusion_matrix(Y_true, [a > 0.5 for a in Y_pred]) fpr, tpr, _ = roc_curve(Y_true, Y_pred) roc_auc = roc_auc_score(Y_true, Y_pred) plt.figure() plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC - %s' % fname.split('/')[-1]) plt.legend(loc="lower right") plt.show() plt.savefig(fname + ' - roc.png') return plt
def gen_report(M, name, obj, err, L_test, S_test, L_true, S_true, y_true): lam = 1.0/np.sqrt(np.max(M.shape)) nn = norm(L_test, 'nuc') on = np.sum(np.abs(S_test)) o = nn + lam*on print('Rank = %d, NNZs = %d' % (matrix_rank(L_test), np.count_nonzero(S_test))) print('Nuclear Norm = %e' % nn) print('One Norm = %e' % on) print('Objective = %e' % o) if L_true is not None: print('Recovery Error = %e' % (norm(L_test - L_true, 'fro')/norm(L_true, 'fro'))) if y_true is not None: y_test = np.linalg.norm(S_test, axis=1) tp, fp, _ = metrics.roc_curve(y_true, y_test) score = metrics.roc_auc_score(y_true, y_test) auc_ax.plot(tp, fp, label=name + ' AUC=' + str(score)) obj_ax.plot(obj, label=name + ' Objective')
def train(self): # data preparation bl_strings = self.prep.load_url_file(TRAINING_DATA_BLACKLIST_FILEPATH, skip_lines=3) wl_strings = self.prep.load_url_file(TRAINING_DATA_WHITELIST_FILEPATH, skip_lines=3) url_strings = bl_strings + wl_strings X = self.prep.to_one_hot_array(url_strings) Y = np.concatenate( [ np.ones(len(bl_strings)), np.zeros(len(wl_strings)) ]) self.net = text_cnn(self.prep.max_index , self.prep.max_len) # model training (X_train, X_test,Y_train,Y_test) = self.prep.train_test_split(X,Y,.5) self.net.fit(X_train, Y_train, batch_size=128, epochs=25) #model evaluation Y_pred = self.net.predict(X_test) fpr, tpr, thresholds = roc_curve(Y_test, Y_pred) auc_score = auc(fpr,tpr) print "\n AUC Score: " + str(auc_score) + "\n"
def run_test(model, x_test_3d, x_test_f, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_test_3d[:, 0], x_test_3d[:, 1], x_test_f[:, 0], x_test_f[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_multi_endo.txt', 'a') target.write("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create sets of 5 with 4 in training and 1 in test
def run_test(model, x_test_3d, x_test_f, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_test_3d[:, 0], x_test_3d[:, 1], x_test_f[:, 0], x_test_f[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_multi_epi.txt', 'a') target.write("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create sets of 5 with 4 in training and 1 in test
def run_test(model, x_ts, y_ts, tr1_n, tr2_n, ts_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary.txt', 'a') target.write("epi, trained on: (" + str(tr1_n) + ", " + str(tr2_n) + ") , tested on: " + str(ts_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("epi, trained on: (" + str(tr1_n) + ", " + str(tr2_n) + ") , tested on: " + str(ts_n) + ", auc: " + str(roc_auc) + "\n") # load 1 and 2 and test on 3
def run_test(model, x_ts, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_endo.txt', 'a') target.write("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create groups of 4 image sets as training and 1 as test
def run_test(model, x_ts, y_ts, tr1_n, tr2_n, ts_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary.txt', 'a') target.write("endo, trained on: (" + str(tr1_n) + ", " + str(tr2_n) + ") , tested on: " + str(ts_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("endo, trained on: (" + str(tr1_n) + ", " + str(tr2_n) + ") , tested on: " + str(ts_n) + ", auc: " + str(roc_auc) + "\n") # load 1 and 2 and test on 3
def run_test(model, x_ts, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_epi.txt', 'a') target.write("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create groups of 4 image sets as training and 1 as test
def run_test(model, x_ts, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_model1.txt', 'a') target.write("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("endo, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create groups of 4 image sets as training and 1 as test
def run_test(model, x_ts, y_ts, tr_ids, ts_n, conv_n, dense_n): # compute final accuracy on training and test sets pred_ts = model.predict([x_ts[:, 0], x_ts[:, 1]]) # get auc scores tpr, fpr, _ = roc_curve(y_ts, pred_ts) roc_auc = auc(fpr, tpr) target = open('auc_scores_summary_epi_deep.txt', 'a') target.write("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") target.close() print("epi, trained on: " + str(tr_ids) + ", tested on: " + str(ts_n) + ", conv n: " + str(conv_n) + ", dense n: " + str(dense_n) + ", auc: " + str(roc_auc) + "\n") # create groups of 4 image sets as training and 1 as test
def test_on_SUP_model(sup_model_name, x, y): SUP_MODEL = load_model(sup_model_name, custom_objects={"contrastive_loss": contrastive_loss}) model_pred = SUP_MODEL.predict([x[:, 0], x[:, 1]]) tpr, fpr, _ = roc_curve(y, model_pred) roc_auc = auc(fpr, tpr) print('auc is : ' + str(roc_auc)) plt.figure(1) plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc) plt.hold(True) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC curve') plt.legend(loc="lower right") plt.hold(False) plt.savefig('/home/nripesh/Dropbox/temp_images/nnet_train_images/roc_curve_siamese_unsup.png')
def test_on_UNSUP_model(unsup_model_name, x, y, roc_curv_save_name): UNSUP_ENCODER = load_model(unsup_model_name) x_0_encode = UNSUP_ENCODER.predict(x[:, 0, :, 1:, 1:, 1:]) x_1_encode = UNSUP_ENCODER.predict(x[:, 1, :, 1:, 1:, 1:]) # vectorize the matrices x_en_sz = x_0_encode.shape x_0_encode = np.reshape(x_0_encode, (x_en_sz[0], x_en_sz[1] * x_en_sz[2] * x_en_sz[3] * x_en_sz[4])) x_1_encode = np.reshape(x_1_encode, (x_en_sz[0], x_en_sz[1] * x_en_sz[2] * x_en_sz[3] * x_en_sz[4])) model_pred = dist_calc_simple(x_0_encode, x_1_encode) tpr, fpr, _ = roc_curve(y, model_pred) roc_auc = auc(fpr, tpr) print('auc is : ' + str(roc_auc)) # plt.figure(2) # plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc) # plt.hold(True) # plt.plot([0, 1], [0, 1], 'k--') # plt.xlim([0.0, 1.0]) # plt.ylim([0.0, 1.05]) # plt.xlabel('False Positive Rate') # plt.ylabel('True Positive Rate') # plt.title('ROC curve') # plt.legend(loc="lower right") # plt.hold(False) # plt.savefig('/home/nripesh/Dropbox/temp_images/nnet_train_images/' + roc_curv_save_name + '.png')
def plot_roc_curve(test_target, pred_score): # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() fpr['micro'], tpr['micro'], _ = metrics.roc_curve(test_target, np.max(pred_score, axis=1)) roc_auc['micro'] = metrics.auc(fpr['micro'], tpr['micro']) # Plot of a ROC curve for a specific class plt.figure() plt.plot(fpr['micro'], tpr['micro'], label='micro-average ROC curve (area = {0:0.2f})' ''.format(metrics.roc_auc['micro'])) # plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % metrics.auc(fpr, tpr)) plt.plot([0, 1], [0, 1], 'k--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc="lower right") plt.show(block=True) # for i in range(n_classes): # fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) # roc_auc[i] = auc(fpr[i], tpr[i]) # python metrics.py --classifier ~/data/results/plot/classifier.h5
def auc(y,yp,pos=1,draw=False): fpr,tpr,thresholds = metrics.roc_curve(y,yp,pos_label=pos) score = metrics.auc(fpr,tpr) if draw: import matplotlib.pyplot as plt plt.title('Receiver Operating Characteristic') plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % score) plt.legend(loc = 'lower right') plt.plot([0, 1], [0, 1],'r--') plt.xlim([0, 1]) plt.ylim([0, 1]) plt.ylabel('True Positive Rate') plt.xlabel('False Positive Rate') plt.show() return score
def getFPRandTPR(self,X,Y): probas_ = self.classifier.fit(self.train_X, self.train_Y).predict_proba(X) fpr, tpr, thresholds = roc_curve(Y, probas_[:, 1]) return fpr, tpr
def getAUCOneTask(self,test_tasks,t): global eta_global X_t, Y_t = self.extractTaskData(self.train_tasks,t) X_test_t, Y_test_t = self.extractTaskData(test_tasks, t) overallKernel = self.constructKernelFunction(t) self.classifiers[t] = SVC(C=self.C, kernel=overallKernel, probability=True, max_iter=self.max_iter_internal, tol=self.tolerance) probas_ = self.classifiers[t].fit(X_t, Y_t).predict_proba(X_test_t) fpr, tpr, thresholds = roc_curve(Y_test_t, probas_[:, 1]) return auc(fpr, tpr), fpr, tpr
def run_model(model): '''Train model''' # Call global variables x_train, x_test, y_train, y_test = X_TRAIN, X_TEST, Y_TRAIN, Y_TEST model.fit(x_train, y_train) # make predictions for test data y_pred = model.predict(x_test) # Accuracy acc = metrics.accuracy_score(y_test, y_pred) print('Accuracy: %.2f%%' % (acc * 100.0)) # F1_score # f1_score = metrics.f1_score(y_test, y_pred) # print("F1_score: %.2f%%" % (f1_score * 100.0)) # AUC of ROC fpr, tpr, _ = metrics.roc_curve(y_test, y_pred) auc = metrics.auc(fpr, tpr) print('AUC: %.3f' % (auc)) # Logs for each fold crossvalidation_acc.append(acc) crossvalidation_auc.append(auc) if ARGS.m: cnf_matrix = confusion_matrix(y_test, y_pred) print(cnf_matrix) np.set_printoptions(precision=2) if ARGS.t == '2': classes = np.asarray(['Spiced', 'Non-spliced']) plot_confusion_matrix(cnf_matrix, classes=classes, normalize=True) elif ARGS.t == '3': classes = np.asarray(['Low', 'Medium', 'High']) plot_confusion_matrix(cnf_matrix, classes=classes, normalize=True) plt.show() if ARGS.f: feature_selection(imp=IMP, model=model) print()
def get_metrics(truth, predicted): """Compute evaluation metrics for a given fold Args: truth [list]: List of labels predicted [list]: List of predicted scores Returns: eval_metrics [dict]: dict of metrics to be saved in the db """ eval_metrics = {} fpr, tpr, thresholds = metrics.roc_curve(truth, predicted) eval_metrics["fpr"] = fpr eval_metrics["tpr"] = tpr eval_metrics["thresholds"] = thresholds eval_metrics["auc"] = metrics.auc(fpr, tpr) eval_metrics["precision"] = {} # TODO eval_metrics["recall"] = {} # TODO eval_metrics["f1"] = {} # TODO for threshold in THRESHOLDS: precision, recall, f1 = precision_recall_at_x_proportion(truth, predicted, x_proportion=threshold/100) eval_metrics.update({threshold: {'precision': precision, 'recall': recall, 'f1': f1}}) return eval_metrics
