make new indexes and added pfroc

This commit is contained in:
Stefan 2022-04-25 10:22:45 +02:00
parent 5d4eaf6e28
commit cd86205896
6 changed files with 53 additions and 26 deletions

View File

@ -38,7 +38,9 @@ parser.add_argument('--series', '-s',
help='List of series to include, must correspond with' + help='List of series to include, must correspond with' +
"path files in ./data/") "path files in ./data/")
parser.add_argument('-experiment', parser.add_argument('-experiment',
help='add experiment title to store the files correctly: test_b50_b400_b800' help='add experiment title to store the files correctly: test_b50_b400_b800')
parser.add_argument('-fold',
help='import fold'
) )
args = parser.parse_args() args = parser.parse_args()
@ -141,7 +143,8 @@ for img_idx in tqdm(range(num_images)): #[:40]): #for less images
# train_idxs = list(train_idxs) # train_idxs = list(train_idxs)
# valid_idxs = list(valid_idxs) # valid_idxs = list(valid_idxs)
yml_paths = read_yaml_to_dict('./../data/Nijmegen paths/train_val_test_idxs.yml') yml_paths = read_yaml_to_dict(f'./../data/Nijmegen paths/train_val_test_idxs_{args.fold}.yml')
print('test, train paths',yml_paths)
train_idxs = yml_paths['train_set0'] train_idxs = yml_paths['train_set0']
valid_idxs = yml_paths['val_set0'] valid_idxs = yml_paths['val_set0']

View File

@ -104,4 +104,4 @@ if __name__ == '__main__':
if type(data_dict[key]) == list: if type(data_dict[key]) == list:
print(f"{key}: {len(data_dict[key])}") print(f"{key}: {len(data_dict[key])}")
dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs", verbose=False) dump_dict_to_yaml(data_dict, "./../data", filename=f"train_val_test_idxs_0", verbose=False)

View File

@ -3,6 +3,7 @@ from sfransen.utils_quintin import *
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import argparse import argparse
import matplotlib.ticker as tkr import matplotlib.ticker as tkr
from sfransen.FROC.p_auc import partial_auc
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Visualise froc results') description='Visualise froc results')
@ -28,6 +29,7 @@ print(experiments)
experiment_path = [] experiment_path = []
experiment_metrics = {} experiment_metrics = {}
auroc = [] auroc = []
paufroc = []
fig = plt.figure(1) fig = plt.figure(1)
ax = fig.add_subplot(111) ax = fig.add_subplot(111)
@ -35,6 +37,9 @@ for idx in range(len(args.experiment)):
experiment_path = f'./../train_output/{experiments[idx]}/froc_metrics_focal_10.yml' experiment_path = f'./../train_output/{experiments[idx]}/froc_metrics_focal_10.yml'
experiment_metrics = read_yaml_to_dict(experiment_path) experiment_metrics = read_yaml_to_dict(experiment_path)
pfroc = partial_auc(experiment_metrics["sensitivity"],experiment_metrics["FP_per_case"])
paufroc.append(round(pfroc,2)
plt.plot(experiment_metrics["FP_per_case"], experiment_metrics["sensitivity"],color=colors[idx],linestyle=plot_type[idx]) plt.plot(experiment_metrics["FP_per_case"], experiment_metrics["sensitivity"],color=colors[idx],linestyle=plot_type[idx])
ax.set(xscale="log") ax.set(xscale="log")
ax.axes.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) ax.axes.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
@ -58,11 +63,14 @@ experiments = [exp.replace('train_n0.001_', '') for exp in experiments]
experiments = [exp.replace('_', ' ') for exp in experiments] experiments = [exp.replace('_', ' ') for exp in experiments]
# experiments = ['10% noise','1% noise','0.1% noise','0.05% noise'] # experiments = ['10% noise','1% noise','0.1% noise','0.05% noise']
concat_func = lambda x,y: x + " (" + str(y) + ")"
experiments_paufroc = list(map(concat_func,experiments,paufroc)) # list the map function
plt.figure(1) plt.figure(1)
plt.title('fROC curve') plt.title('fROC curve')
plt.xlabel('False positive per case') plt.xlabel('False positive per case')
plt.ylabel('Sensitivity') plt.ylabel('Sensitivity')
plt.legend(experiments,loc='lower right') plt.legend(experiments_paufroc,loc='lower right')
# plt.xlim([0,50]) # plt.xlim([0,50])
plt.grid() plt.grid()
plt.ylim([0,1]) plt.ylim([0,1])

View File

@ -2,33 +2,23 @@ import matplotlib.pyplot as plt
import matplotlib.ticker as tkr import matplotlib.ticker as tkr
import seaborn as sns import seaborn as sns
import matplotlib.ticker as tkr import matplotlib.ticker as tkr
from p_auc import partial_auc
x = [0,1.1,2.3,5,90,100,1500] x = [0,0.11,0.23,0.5,0.90,1.00,1.500,3]
y = [0,0.02,0.09,1,2,3,4] y = [0,0.02,0.09,1,2,3,4,12]
tick_spacing = 1 tick_spacing = 1
fig1, ax1 = plt.subplots(1,1) fig1, ax1 = plt.subplots(1,1)
ax1.plot(x,y) ax1.plot(x,y)
# ax.set_xticks([0,100,1500]) pauc = partial_auc(x,y)
# ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing)) print(pauc)
ax1.set(xscale="log")
ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
ax1.xaxis.set_minor_formatter(tkr.NullFormatter())
ax1.xaxis.set_major_formatter(tkr.ScalarFormatter())
ax1.grid(True, which="both", ls="--", c='#d3d3d3')
ax1.set_xlim(left=0, right=150)
ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3]))
fig2, ax2 = plt.subplots(1,1)
ax2.plot(x,y)
# ax.set_xticks([0,100,1500]) # ax.set_xticks([0,100,1500])
# ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing)) # ax.xaxis.set_major_locator(ticker.MultipleLocator(tick_spacing))
ax2.set(xscale="log") # ax1.set(xscale="log")
ax2.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all')) # ax1.xaxis.set_minor_locator(tkr.LogLocator(base=10, subs='all'))
ax2.xaxis.set_minor_formatter(tkr.NullFormatter()) # ax1.xaxis.set_minor_formatter(tkr.NullFormatter())
ax2.xaxis.set_major_formatter(tkr.ScalarFormatter()) # ax1.xaxis.set_major_formatter(tkr.ScalarFormatter())
ax2.grid(True, which="both", ls="--", c='#d3d3d3') # ax1.grid(True, which="both", ls="--", c='#d3d3d3')
ax2.set_xlim(left=0, right=150) # ax1.set_xlim(left=0, right=150)
ax2.xaxis.set_major_locator(tkr.FixedLocator([0,1,30,500])) # ax1.xaxis.set_major_locator(tkr.FixedLocator([0,1,3]))
plt.show()

View File

@ -4,3 +4,4 @@ from .data_utils import *
from .cal_froc_from_np import * from .cal_froc_from_np import *
from .blob_preprocess import * from .blob_preprocess import *
from .analysis_utils import * from .analysis_utils import *
from .p_auc import *

25
src/sfransen/FROC/p_auc.py Executable file
View File

@ -0,0 +1,25 @@
import numpy as np
def partial_auc(sensitivity, fp_per_patient, low=0.1, high=2.5):
"""
Calculates partial area-under-the-curve from FROC sensitivity/FPP curve.
"""
# Remove all values above min and max FPP
clipped_sens, clipped_fpp = [0.], [0.]
max_sens_before_window = 0
max_fpp_before_window = 0
for sens, fpp in zip(sensitivity, fp_per_patient):
if fpp < low and fpp > max_fpp_before_window:
max_sens_before_window = sens
if fpp >= low and fpp <= high:
clipped_sens.append(sens)
clipped_fpp.append(fpp)
# Extend the window to the limits supplied by the user
clipped_sens = [max_sens_before_window] + clipped_sens + [clipped_sens[-1]]
clipped_fpp = [low] + clipped_fpp + [high]
# Integrate y(x) with the trapezoid rule
return np.trapz(clipped_sens, clipped_fpp)