Skip to content
Snippets Groups Projects
Commit 71b89893 authored by Nikolai.Hartmann's avatar Nikolai.Hartmann
Browse files

Merge remote-tracking branch 'origin/master'

parents ae08f994 e543ec8d
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/env python
import os
import argparse
import keras
import h5py
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
import numpy as np
from KerasROOTClassification import ClassificationProject
parser = argparse.ArgumentParser(description='Evaluate a model from a classification project using the given '
'weights and plot the ROC curve and train/test overlayed scores')
parser.add_argument("project_dir")
parser.add_argument("weights")
parser.add_argument("-p", "--plot-prefix", default="eval_nn")
args = parser.parse_args()
c = ClassificationProject(args.project_dir)
c.model.load_weights(args.weights)
print("Predicting for test sample ...")
scores_test = c.evaluate(c.x_test)
print("Done")
fpr, tpr, threshold = roc_curve(c.y_test, scores_test, sample_weight = c.w_test)
fpr = 1.0 - fpr
try:
roc_auc = auc(tpr, fpr, reorder=True)
except ValueError:
logger.warning("Got a value error from auc - trying to rerun with reorder=True")
roc_auc = auc(tpr, fpr, reorder=True)
plt.grid(color='gray', linestyle='--', linewidth=1)
plt.plot(tpr, fpr, label=str(c.name + " (AUC = {})".format(roc_auc)))
plt.plot([0,1],[1,0], linestyle='--', color='black', label='Luck')
plt.ylabel("Background rejection")
plt.xlabel("Signal efficiency")
plt.title('Receiver operating characteristic')
plt.xlim(0,1)
plt.ylim(0,1)
plt.xticks(np.arange(0,1,0.1))
plt.yticks(np.arange(0,1,0.1))
plt.legend(loc='lower left', framealpha=1.0)
plt.savefig(args.plot_prefix+"_ROC.pdf")
plt.clf()
......@@ -12,7 +12,6 @@ else:
import os
import json
import yaml
import pickle
import importlib
import csv
......@@ -44,15 +43,15 @@ from .utils import WeightedRobustScaler, weighted_quantile
# configure number of cores
# this doesn't seem to work, but at least with these settings keras only uses 4 processes
import tensorflow as tf
from keras import backend as K
num_cores = 1
config = tf.ConfigProto(intra_op_parallelism_threads=num_cores,
inter_op_parallelism_threads=num_cores,
allow_soft_placement=True,
device_count = {'CPU': num_cores})
session = tf.Session(config=config)
K.set_session(session)
# import tensorflow as tf
# from keras import backend as K
# num_cores = 1
# config = tf.ConfigProto(intra_op_parallelism_threads=num_cores,
# inter_op_parallelism_threads=num_cores,
# allow_soft_placement=True,
# device_count = {'CPU': num_cores})
# session = tf.Session(config=config)
# K.set_session(session)
import ROOT
......@@ -642,7 +641,7 @@ class ClassificationProject(object):
# the (other) hidden layers
for node_count, dropout_fraction in zip(self.nodes[start_layer:], self.dropout[start_layer:]):
self._model.add(Dense(node_count, activation=self.activation_function))
if dropout_fraction > 0:
if (dropout_fraction is not None) and (dropout_fraction > 0):
self._model.add(Dropout(rate=dropout_fraction))
# last layer is one neuron (binary classification)
self._model.add(Dense(1, activation=self.activation_function_output))
......@@ -999,27 +998,32 @@ class ClassificationProject(object):
plt.close(fig)
def plot_ROC(self):
def plot_ROC(self, xlim=(0,1), ylim=(0,1)):
logger.info("Plot ROC curve")
fpr, tpr, threshold = roc_curve(self.y_test, self.scores_test, sample_weight = self.w_test)
fpr = 1.0 - fpr
try:
roc_auc = auc(tpr, fpr, reorder=True)
except ValueError:
logger.warning("Got a value error from auc - trying to rerun with reorder=True")
roc_auc = auc(tpr, fpr, reorder=True)
plt.grid(color='gray', linestyle='--', linewidth=1)
plt.plot(tpr, fpr, label=str(self.name + " (AUC = {})".format(roc_auc)))
for y, scores, weight, label in [
(self.y_train, self.scores_train, self.w_train, "train"),
(self.y_test, self.scores_test, self.w_test, "test")
]:
fpr, tpr, threshold = roc_curve(y, scores, sample_weight = weight)
fpr = 1.0 - fpr # background rejection
try:
roc_auc = auc(tpr, fpr)
except ValueError:
logger.warning("Got a value error from auc - trying to rerun with reorder=True")
roc_auc = auc(tpr, fpr, reorder=True)
plt.plot(tpr, fpr, label=str(self.name + " {} (AUC = {:.3f})".format(label, roc_auc)))
plt.plot([0,1],[1,0], linestyle='--', color='black', label='Luck')
plt.ylabel("Background rejection")
plt.xlabel("Signal efficiency")
plt.title('Receiver operating characteristic')
plt.xlim(0,1)
plt.ylim(0,1)
plt.xticks(np.arange(0,1,0.1))
plt.yticks(np.arange(0,1,0.1))
plt.xlim(*xlim)
plt.ylim(*ylim)
# plt.xticks(np.arange(0,1,0.1))
# plt.yticks(np.arange(0,1,0.1))
plt.legend(loc='lower left', framealpha=1.0)
plt.savefig(os.path.join(self.project_dir, "ROC.pdf"))
plt.clf()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment