Skip to content

Commit 41b891e

Browse files
author
dbouget
committed
Potential issue with extra metrics, investigating [skip ci]
1 parent ed8ca80 commit 41b891e

File tree

4 files changed

+66
-65
lines changed

4 files changed

+66
-65
lines changed

‎raidionicsval/Utils/PatientMetricsStructure.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -239,28 +239,28 @@ def setup_extra_metrics(self, metric_names):
239239
240240
"""
241241
thr_list = self._class_metrics[self.class_names[0]].get_probability_thresholds_list()
242-
complete_metric_names = []
243-
for m in metric_names:
244-
complete_metric_names.extend([f'PiW {m}', f'OW {m}'])
242+
# complete_metric_names = []
243+
# for m in metric_names:
244+
# complete_metric_names.extend([f'PiW {m}', f'OW {m}'])
245245
if self._extra_metrics is None:
246246
self._extra_metrics = []
247247
for thr in thr_list:
248248
curr_thr = [thr]
249-
for m in complete_metric_names:
249+
for m in metric_names:
250250
curr_thr.append([m, float('nan')])
251251
self._extra_metrics.append(curr_thr)
252252
else:
253253
existing_metrics = [x[0] for x in self._extra_metrics[0][1:]]
254-
matching_metrics_states = all(element in existing_metrics for element in complete_metric_names)
254+
matching_metrics_states = all(element in existing_metrics for element in metric_names)
255255
if not matching_metrics_states:
256-
for m in complete_metric_names:
256+
for m in metric_names:
257257
if m not in existing_metrics:
258258
for th in range(len(self._extra_metrics)):
259259
self._extra_metrics[th].append([m, float('nan')])
260260

261261
# Performs the same operation on the extra metrics for each class
262262
for cl in self._class_names:
263-
self._class_metrics[cl].setup_extra_metrics(complete_metric_names)
263+
self._class_metrics[cl].setup_extra_metrics(metric_names)
264264

265265

266266
class ClassMetrics:

‎raidionicsval/Utils/resources.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ def __setup(self):
3434
self.config = None
3535
self.home_path = ''
3636

37-
self.upper_default_metrics_index = 23 # Single place for holding this attribute, "safer" approach, column index after # GT and # Det
37+
self.upper_default_metrics_index = 23 # Single place for holding this attribute, "safer" approach, column index after # GT and # Det
3838

3939
self.data_root = ""
4040
self.task = None

‎raidionicsval/Validation/extra_metrics_computation.py‎

Lines changed: 57 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -115,64 +115,65 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold
115115
obj_val.run()
116116
# Computing all metrics in an object-wise fashion
117117
all_instance_results = []
118-
for g, go in enumerate(obj_val.gt_candidates):
119-
gt_label = g + 1
120-
if gt_label in np.asarray(obj_val.matching_results)[:, 0]:
121-
indices = np.where(np.asarray(obj_val.matching_results)[:, 0] == gt_label)[0]
122-
if len(indices) > 1:
123-
# Should not happen anymore
124-
print(f"Warning - Entering a use-case which should not be possible!")
125-
pass
126-
det_label = np.asarray(obj_val.matching_results)[indices[0]][1]
127-
# det_label = obj_val.matching_results[list(np.asarray(obj_val.matching_results)[:, 0]).index(gt_label)][1]
128-
instance_gt_array = np.zeros(gt.shape, dtype="uint8")
129-
instance_det_array = np.zeros(detection.shape, dtype="uint8")
130-
instance_gt_array[obj_val.gt_labels == gt_label] = 1
131-
instance_det_array[obj_val.detection_labels == det_label] = 1
132-
133-
tp_array = np.zeros(instance_gt_array.shape)
134-
fp_array = np.zeros(instance_gt_array.shape)
135-
tn_array = np.zeros(instance_gt_array.shape)
136-
fn_array = np.zeros(instance_gt_array.shape)
137-
138-
tp_array[(instance_gt_array == 1) & (instance_det_array == 1)] = 1
139-
fp_array[(instance_gt_array == 0) & (instance_det_array == 1)] = 1
140-
tn_array[(instance_gt_array == 0) & (instance_det_array == 0)] = 1
141-
fn_array[(instance_gt_array == 1) & (instance_det_array == 0)] = 1
142-
instance_results = []
143-
if SharedResources.getInstance().number_processes > 1:
144-
try:
145-
pool = multiprocessing.Pool(processes=SharedResources.getInstance().number_processes)
146-
instance_results = pool.map(parallel_metric_computation, zip(metrics, metric_values,
147-
itertools.repeat(instance_gt_array),
148-
itertools.repeat(instance_det_array),
149-
itertools.repeat(
150-
det_input_specs[1]),
151-
itertools.repeat(
152-
gt_input_specs[1]),
153-
itertools.repeat(tp_array),
154-
itertools.repeat(tn_array),
155-
itertools.repeat(fp_array),
156-
itertools.repeat(fn_array)))
157-
pool.close()
158-
pool.join()
159-
except Exception as e:
160-
print("Issue computing metrics for patient {} in the multiprocessing loop.".format(
161-
patient_object.unique_id))
162-
print(traceback.format_exc())
163-
else:
164-
for metric in metrics:
118+
if len(obj_val.matching_results) != 0:
119+
for g, go in enumerate(obj_val.gt_candidates):
120+
gt_label = g + 1
121+
if gt_label in np.asarray(obj_val.matching_results)[:, 0]:
122+
indices = np.where(np.asarray(obj_val.matching_results)[:, 0] == gt_label)[0]
123+
if len(indices) > 1:
124+
# Should not happen anymore
125+
print(f"Warning - Entering a use-case which should not be possible!")
126+
pass
127+
det_label = np.asarray(obj_val.matching_results)[indices[0]][1]
128+
# det_label = obj_val.matching_results[list(np.asarray(obj_val.matching_results)[:, 0]).index(gt_label)][1]
129+
instance_gt_array = np.zeros(gt.shape, dtype="uint8")
130+
instance_det_array = np.zeros(detection.shape, dtype="uint8")
131+
instance_gt_array[obj_val.gt_labels == gt_label] = 1
132+
instance_det_array[obj_val.detection_labels == det_label] = 1
133+
134+
tp_array = np.zeros(instance_gt_array.shape)
135+
fp_array = np.zeros(instance_gt_array.shape)
136+
tn_array = np.zeros(instance_gt_array.shape)
137+
fn_array = np.zeros(instance_gt_array.shape)
138+
139+
tp_array[(instance_gt_array == 1) & (instance_det_array == 1)] = 1
140+
fp_array[(instance_gt_array == 0) & (instance_det_array == 1)] = 1
141+
tn_array[(instance_gt_array == 0) & (instance_det_array == 0)] = 1
142+
fn_array[(instance_gt_array == 1) & (instance_det_array == 0)] = 1
143+
instance_results = []
144+
if SharedResources.getInstance().number_processes > 1:
165145
try:
166-
metric_value = compute_specific_metric_value(metric=metric, gt=instance_gt_array, detection=instance_det_array,
167-
tp=np.sum(tp_array), tn=np.sum(tn_array),
168-
fp=np.sum(fp_array), fn=np.sum(fn_array),
169-
gt_spacing=gt_input_specs[1],
170-
det_spacing=det_input_specs[1])
171-
instance_results.append([metric, metric_value])
146+
pool = multiprocessing.Pool(processes=SharedResources.getInstance().number_processes)
147+
instance_results = pool.map(parallel_metric_computation, zip(metrics, metric_values,
148+
itertools.repeat(instance_gt_array),
149+
itertools.repeat(instance_det_array),
150+
itertools.repeat(
151+
det_input_specs[1]),
152+
itertools.repeat(
153+
gt_input_specs[1]),
154+
itertools.repeat(tp_array),
155+
itertools.repeat(tn_array),
156+
itertools.repeat(fp_array),
157+
itertools.repeat(fn_array)))
158+
pool.close()
159+
pool.join()
172160
except Exception as e:
173-
print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id))
161+
print("Issue computing metrics for patient {} in the multiprocessing loop.".format(
162+
patient_object.unique_id))
174163
print(traceback.format_exc())
175-
all_instance_results.append(instance_results)
164+
else:
165+
for metric in metrics:
166+
try:
167+
metric_value = compute_specific_metric_value(metric=metric, gt=instance_gt_array, detection=instance_det_array,
168+
tp=np.sum(tp_array), tn=np.sum(tn_array),
169+
fp=np.sum(fp_array), fn=np.sum(fn_array),
170+
gt_spacing=gt_input_specs[1],
171+
det_spacing=det_input_specs[1])
172+
instance_results.append([metric, metric_value])
173+
except Exception as e:
174+
print('Issue computing metric {} for patient {}'.format(metric, patient_object.unique_id))
175+
print(traceback.format_exc())
176+
all_instance_results.append(instance_results)
176177

177178
if len(all_instance_results) != 0:
178179
for m in metrics:
@@ -193,7 +194,7 @@ def compute_patient_extra_metrics(patient_object, class_index, optimal_threshold
193194
extra_metrics_results.append([f'OW {m}', tp_mean])
194195
else:
195196
for m in metrics:
196-
extra_metrics_results.append([f'OW {m}', -999.])
197+
extra_metrics_results.append([f'OW {m}', math.inf])
197198
except Exception as e:
198199
print('Global issue computing metrics for patient {}'.format(patient_object.unique_id))
199200
print(traceback.format_exc())

‎raidionicsval/Validation/validation_utilities.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ def compute_singe_fold_average_metrics(results, fold_number, best_threshold, bes
302302
extra_metrics_std = []
303303
for m in metric_names:
304304
if m in fold_results.columns.values:
305-
if m in ['HD95', 'ASSD', 'RAVD', 'VC', 'OASSD']:
305+
if m.split(' ')[1] in ['HD95', 'ASSD', 'RAVD', 'VC', 'OASSD']:
306306
avg = fold_results.loc[thresh_index][fold_results.loc[thresh_index][m] != -1.0][m].dropna().astype(
307307
'float32').mean()
308308
std = fold_results.loc[thresh_index][fold_results.loc[thresh_index][m] != -1.0][m].dropna().astype(

0 commit comments

Comments
 (0)