make new indexes and added pfroc

scripts/1.U-net_chris.py

@@ -38,7 +38,9 @@ parser.add_argument('--series', '-s',
     help='List of series to include, must correspond with' +
         "path files in ./data/")
 parser.add_argument('-experiment',
-    help='add experiment title to store the files correctly: test_b50_b400_b800' 
+    help='add experiment title to store the files correctly: test_b50_b400_b800') 
+parser.add_argument('-fold',
+    help='import fold' 
 )
 args = parser.parse_args()
 
@@ -141,7 +143,8 @@ for img_idx in tqdm(range(num_images)): #[:40]): #for less images
 # train_idxs = list(train_idxs)
 # valid_idxs = list(valid_idxs)
 
-yml_paths = read_yaml_to_dict('./../data/Nijmegen paths/train_val_test_idxs.yml')
+yml_paths = read_yaml_to_dict(f'./../data/Nijmegen paths/train_val_test_idxs_{args.fold}.yml')
+print('test, train paths',yml_paths)
 train_idxs = yml_paths['train_set0']
 valid_idxs = yml_paths['val_set0']
 

scripts/3.make_train_val_test_indexes.py

@@ -104,4 +104,4 @@ if __name__ == '__main__':
         if type(data_dict[key]) == list:
             print(f"{key}: {len(data_dict[key])}")
 
-    dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs", verbose=False)
+    dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs_0", verbose=False)

scripts/5.Visualize_frocs.py

@@ -3,6 +3,7 @@ from sfransen.utils_quintin import *
 import matplotlib.pyplot as plt
 import argparse
 import matplotlib.ticker as tkr
+from sfransen.FROC.p_auc import partial_auc
 
 parser = argparse.ArgumentParser(
     description='Visualise froc results')
@@ -28,6 +29,7 @@ print(experiments)
 experiment_path = []
 experiment_metrics = {}
 auroc = []
+paufroc = []
 
 fig = plt.figure(1)
 ax = fig.add_subplot(111)
@@ -35,6 +37,9 @@ for idx in range(len(args.experiment)):
     experiment_path = f'./../train_output/{experiments[idx]}/froc_metrics_focal_10.yml'
     experiment_metrics = read_yaml_to_dict(experiment_path)
 
+    pfroc = partial_auc(experiment_metrics["sensitivity"],experiment_metrics["FP_per_case"])
+    paufroc.append(round(pfroc,2)
+
     plt.plot(experiment_metrics["FP_per_case"], experiment_metrics["sensitivity"],color=colors[idx],linestyle=plot_type[idx])
     ax.set(xscale="log")
     ax.axes.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
@@ -58,11 +63,14 @@ experiments = [exp.replace('train_n0.001_', '') for exp in experiments]
 experiments = [exp.replace('_', ' ') for exp in experiments] 
 # experiments = ['10% noise','1% noise','0.1% noise','0.05% noise']
 
+concat_func = lambda x,y: x + " (" + str(y) + ")"
+experiments_paufroc = list(map(concat_func,experiments,paufroc)) # list the map function
+
 plt.figure(1)
 plt.title('fROC curve')
 plt.xlabel('False positive per case')
 plt.ylabel('Sensitivity')
-plt.legend(experiments,loc='lower right')
+plt.legend(experiments_paufroc,loc='lower right')
 # plt.xlim([0,50])
 plt.grid()
 plt.ylim([0,1])

scripts/tmp.py

@@ -2,33 +2,23 @@ import matplotlib.pyplot as plt
 import matplotlib.ticker as tkr
 import seaborn as sns
 import matplotlib.ticker as tkr
-
+from p_auc import partial_auc
-x = [0,1.1,2.3,5,90,100,1500]
+x = [0,0.11,0.23,0.5,0.90,1.00,1.500,3]
-y = [0,0.02,0.09,1,2,3,4]
+y = [0,0.02,0.09,1,2,3,4,12]
 
 tick_spacing = 1
 
 fig1, ax1 = plt.subplots(1,1)
 ax1.plot(x,y)
-# ax.set_xticks([0,100,1500])
+pauc = partial_auc(x,y)
-# ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing))
+print(pauc)
-ax1.set(xscale="log")
-ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
-ax1.xaxis.set_minor_formatter(tkr.NullFormatter())
-ax1.xaxis.set_major_formatter(tkr.ScalarFormatter())
-ax1.grid(True, which="both", ls="--", c='#d3d3d3')
-ax1.set_xlim(left=0, right=150)
-ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3]))
 
-fig2, ax2 = plt.subplots(1,1)
-ax2.plot(x,y)
 # ax.set_xticks([0,100,1500])
 # ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing))
-ax2.set(xscale="log")
+# ax1.set(xscale="log")
-ax2.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
+# ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
-ax2.xaxis.set_minor_formatter(tkr.NullFormatter())
+# ax1.xaxis.set_minor_formatter(tkr.NullFormatter())
-ax2.xaxis.set_major_formatter(tkr.ScalarFormatter())
+# ax1.xaxis.set_major_formatter(tkr.ScalarFormatter())
-ax2.grid(True, which="both", ls="--", c='#d3d3d3')
+# ax1.grid(True, which="both", ls="--", c='#d3d3d3')
-ax2.set_xlim(left=0, right=150)
+# ax1.set_xlim(left=0, right=150)
-ax2.xaxis.set_major_locator(tkr.FixedLocator([0,1,30,500]))
+# ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3]))
-plt.show()

src/sfransen/FROC/__init__.py

@@ -4,3 +4,4 @@ from .data_utils import *
 from .cal_froc_from_np import *
 from .blob_preprocess import *
 from .analysis_utils import *
+from .p_auc import *

src/sfransen/FROC/p_auc.py

@@ -0,0 +1,25 @@
+import numpy as np
+
+def partial_auc(sensitivity, fp_per_patient, low=0.1, high=2.5):
+    """
+    Calculates partial area-under-the-curve from FROC sensitivity/FPP curve.
+    """
+
+    # Remove all values above min and max FPP
+    clipped_sens, clipped_fpp = [0.], [0.]
+    max_sens_before_window = 0
+    max_fpp_before_window = 0
+    for sens, fpp in zip(sensitivity, fp_per_patient):
+        if fpp < low and fpp > max_fpp_before_window:
+            max_sens_before_window = sens
+
+        if fpp >= low and fpp <= high:
+            clipped_sens.append(sens)
+            clipped_fpp.append(fpp)
+
+    # Extend the window to the limits supplied by the user
+    clipped_sens = [max_sens_before_window] + clipped_sens + [clipped_sens[-1]]
+    clipped_fpp  = [low] + clipped_fpp + [high]
+
+    # Integrate y(x) with the trapezoid rule
+    return np.trapz(clipped_sens, clipped_fpp)