From cd8620589644d23c4ea1be12f05f40644150e545 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 25 Apr 2022 10:22:45 +0200 Subject: [PATCH] make new indexes and added pfroc --- scripts/1.U-net_chris.py | 7 +++-- scripts/3.make_train_val_test_indexes.py | 2 +- scripts/5.Visualize_frocs.py | 10 ++++++- scripts/tmp.py | 34 +++++++++--------------- src/sfransen/FROC/__init__.py | 1 + src/sfransen/FROC/p_auc.py | 25 +++++++++++++++++ 6 files changed, 53 insertions(+), 26 deletions(-) create mode 100755 src/sfransen/FROC/p_auc.py diff --git a/scripts/1.U-net_chris.py b/scripts/1.U-net_chris.py index de8ab83..0bff473 100755 --- a/scripts/1.U-net_chris.py +++ b/scripts/1.U-net_chris.py @@ -38,7 +38,9 @@ parser.add_argument('--series', '-s', help='List of series to include, must correspond with' + "path files in ./data/") parser.add_argument('-experiment', - help='add experiment title to store the files correctly: test_b50_b400_b800' + help='add experiment title to store the files correctly: test_b50_b400_b800') +parser.add_argument('-fold', + help='import fold' ) args = parser.parse_args() @@ -141,7 +143,8 @@ for img_idx in tqdm(range(num_images)): #[:40]): #for less images # train_idxs = list(train_idxs) # valid_idxs = list(valid_idxs) -yml_paths = read_yaml_to_dict('./../data/Nijmegen paths/train_val_test_idxs.yml') +yml_paths = read_yaml_to_dict(f'./../data/Nijmegen paths/train_val_test_idxs_{args.fold}.yml') +print('test, train paths',yml_paths) train_idxs = yml_paths['train_set0'] valid_idxs = yml_paths['val_set0'] diff --git a/scripts/3.make_train_val_test_indexes.py b/scripts/3.make_train_val_test_indexes.py index 2c38185..31f27fc 100755 --- a/scripts/3.make_train_val_test_indexes.py +++ b/scripts/3.make_train_val_test_indexes.py @@ -104,4 +104,4 @@ if __name__ == '__main__': if type(data_dict[key]) == list: print(f"{key}: {len(data_dict[key])}") - dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs", verbose=False) \ No newline at end of file + dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs_0", verbose=False) \ No newline at end of file diff --git a/scripts/5.Visualize_frocs.py b/scripts/5.Visualize_frocs.py index 5f4a96f..93a68fc 100755 --- a/scripts/5.Visualize_frocs.py +++ b/scripts/5.Visualize_frocs.py @@ -3,6 +3,7 @@ from sfransen.utils_quintin import * import matplotlib.pyplot as plt import argparse import matplotlib.ticker as tkr +from sfransen.FROC.p_auc import partial_auc parser = argparse.ArgumentParser( description='Visualise froc results') @@ -28,6 +29,7 @@ print(experiments) experiment_path = [] experiment_metrics = {} auroc = [] +paufroc = [] fig = plt.figure(1) ax = fig.add_subplot(111) @@ -35,6 +37,9 @@ for idx in range(len(args.experiment)): experiment_path = f'./../train_output/{experiments[idx]}/froc_metrics_focal_10.yml' experiment_metrics = read_yaml_to_dict(experiment_path) + pfroc = partial_auc(experiment_metrics["sensitivity"],experiment_metrics["FP_per_case"]) + paufroc.append(round(pfroc,2) + plt.plot(experiment_metrics["FP_per_case"], experiment_metrics["sensitivity"],color=colors[idx],linestyle=plot_type[idx]) ax.set(xscale="log") ax.axes.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) @@ -58,11 +63,14 @@ experiments = [exp.replace('train_n0.001_', '') for exp in experiments] experiments = [exp.replace('_', ' ') for exp in experiments] # experiments = ['10% noise','1% noise','0.1% noise','0.05% noise'] +concat_func = lambda x,y: x + " (" + str(y) + ")" +experiments_paufroc = list(map(concat_func,experiments,paufroc)) # list the map function + plt.figure(1) plt.title('fROC curve') plt.xlabel('False positive per case') plt.ylabel('Sensitivity') -plt.legend(experiments,loc='lower right') +plt.legend(experiments_paufroc,loc='lower right') # plt.xlim([0,50]) plt.grid() plt.ylim([0,1]) diff --git a/scripts/tmp.py b/scripts/tmp.py index cfaffb4..93b65bb 100755 --- a/scripts/tmp.py +++ b/scripts/tmp.py @@ -2,33 +2,23 @@ import matplotlib.pyplot as plt import matplotlib.ticker as tkr import seaborn as sns import matplotlib.ticker as tkr - -x = [0,1.1,2.3,5,90,100,1500] -y = [0,0.02,0.09,1,2,3,4] +from p_auc import partial_auc +x = [0,0.11,0.23,0.5,0.90,1.00,1.500,3] +y = [0,0.02,0.09,1,2,3,4,12] tick_spacing = 1 fig1, ax1 = plt.subplots(1,1) ax1.plot(x,y) -# ax.set_xticks([0,100,1500]) -# ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing)) -ax1.set(xscale="log") -ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) -ax1.xaxis.set_minor_formatter(tkr.NullFormatter()) -ax1.xaxis.set_major_formatter(tkr.ScalarFormatter()) -ax1.grid(True, which="both", ls="--", c='#d3d3d3') -ax1.set_xlim(left=0, right=150) -ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3])) +pauc = partial_auc(x,y) +print(pauc) -fig2, ax2 = plt.subplots(1,1) -ax2.plot(x,y) # ax.set_xticks([0,100,1500]) # ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing)) -ax2.set(xscale="log") -ax2.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) -ax2.xaxis.set_minor_formatter(tkr.NullFormatter()) -ax2.xaxis.set_major_formatter(tkr.ScalarFormatter()) -ax2.grid(True, which="both", ls="--", c='#d3d3d3') -ax2.set_xlim(left=0, right=150) -ax2.xaxis.set_major_locator(tkr.FixedLocator([0,1,30,500])) -plt.show() +# ax1.set(xscale="log") +# ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) +# ax1.xaxis.set_minor_formatter(tkr.NullFormatter()) +# ax1.xaxis.set_major_formatter(tkr.ScalarFormatter()) +# ax1.grid(True, which="both", ls="--", c='#d3d3d3') +# ax1.set_xlim(left=0, right=150) +# ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3])) \ No newline at end of file diff --git a/src/sfransen/FROC/__init__.py b/src/sfransen/FROC/__init__.py index 377a94b..52ca980 100755 --- a/src/sfransen/FROC/__init__.py +++ b/src/sfransen/FROC/__init__.py @@ -4,3 +4,4 @@ from .data_utils import * from .cal_froc_from_np import * from .blob_preprocess import * from .analysis_utils import * +from .p_auc import * diff --git a/src/sfransen/FROC/p_auc.py b/src/sfransen/FROC/p_auc.py new file mode 100755 index 0000000..e264395 --- /dev/null +++ b/src/sfransen/FROC/p_auc.py @@ -0,0 +1,25 @@ +import numpy as np + +def partial_auc(sensitivity, fp_per_patient, low=0.1, high=2.5): + """ + Calculates partial area-under-the-curve from FROC sensitivity/FPP curve. + """ + + # Remove all values above min and max FPP + clipped_sens, clipped_fpp = [0.], [0.] + max_sens_before_window = 0 + max_fpp_before_window = 0 + for sens, fpp in zip(sensitivity, fp_per_patient): + if fpp < low and fpp > max_fpp_before_window: + max_sens_before_window = sens + + if fpp >= low and fpp <= high: + clipped_sens.append(sens) + clipped_fpp.append(fpp) + + # Extend the window to the limits supplied by the user + clipped_sens = [max_sens_before_window] + clipped_sens + [clipped_sens[-1]] + clipped_fpp = [low] + clipped_fpp + [high] + + # Integrate y(x) with the trapezoid rule + return np.trapz(clipped_sens, clipped_fpp) \ No newline at end of file