{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Use Case: Unmatched Instances Input"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!pip install -U panoptica > /dev/null #installs numpy and others\n",
"!pip install -U auxiliary > /dev/null"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup Imports"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"from auxiliary.nifti.io import read_nifti\n",
"from rich import print as pprint\n",
"from panoptica import NaiveThresholdMatching, Panoptic_Evaluator, UnmatchedInstancePair"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load Example Data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To demonstrate we use a reference and predicition of spine a segmentation with unmatched instances.\n",
"\n",
"\n",
""
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"(array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n",
" 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8),\n",
" array([ 0, 3, 4, 5, 6, 7, 8, 9, 27, 103, 104, 105, 106,\n",
" 107, 108, 109, 203, 204, 205, 206, 207, 208, 209], dtype=uint8))"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ref_masks = read_nifti(\"./spine_seg/unmatched_instance/ref.nii.gz\")\n",
"pred_masks = read_nifti(\"./spine_seg/unmatched_instance/pred.nii.gz\")\n",
"\n",
"# labels are unmatching\n",
"np.unique(ref_masks), np.unique(pred_masks)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To use your own data please replace the example data with your own data.\n",
"\n",
"In ordner to successfully load your data please use NIFTI files and the following file designation within the folder \"unmatched_instance\": \n",
"\n",
"```panoptica/spine_seg/unmatched_instance/```\n",
"\n",
"- Reference data (\"ref.nii.gz\")\n",
"- Prediction data (\"pred.nii.gz\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run Evaluation"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"────────────────────────────────────────── Thank you for using panoptica ──────────────────────────────────────────\n",
"\n"
],
"text/plain": [
"\u001b[92m────────────────────────────────────────── \u001b[0mThank you for using \u001b[1mpanoptica\u001b[0m\u001b[92m ──────────────────────────────────────────\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" Please support our development by citing \n",
"\n"
],
"text/plain": [
" Please support our development by citing \n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" https://github.com/BrainLesion/panoptica#citation -- Thank you! \n",
"\n"
],
"text/plain": [
" \u001b[4;94mhttps://github.com/BrainLesion/panoptica#citation\u001b[0m -- Thank you! \n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
"\n"
],
"text/plain": [
"\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Panoptic: Start Evaluation\n",
"-- Got UnmatchedInstancePair, will match instances\n",
"-- Got MatchedInstancePair, will evaluate instances\n",
"-- evaluate took 3.694295644760132 seconds to execute.\n"
]
}
],
"source": [
"sample = UnmatchedInstancePair(pred_masks, ref_masks)\n",
"evaluator = Panoptic_Evaluator(\n",
" expected_input=UnmatchedInstancePair,\n",
" instance_matcher=NaiveThresholdMatching(),\n",
")\n",
"\n",
"result, debug_data = evaluator.evaluate(sample)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Inspect Results\n",
"The results object allows access to individual metrics and provides helper methods for further processing"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"+++ MATCHING +++\n",
"Number of instances in reference (num_ref_instances): 22\n",
"Number of instances in prediction (num_pred_instances): 22\n",
"True Positives (tp): 19\n",
"False Positives (fp): 3\n",
"False Negatives (fn): 3\n",
"Recognition Quality / F1-Score (rq): 0.8636363636363636\n",
"\n",
"+++ GLOBAL +++\n",
"Global Binary Dice (global_bin_dsc): 0.9744370224078394\n",
"Global Binary Centerline Dice (global_bin_cldsc): 0.9637064011802574\n",
"\n",
"+++ INSTANCE +++\n",
"Segmentation Quality IoU (sq): 0.8328184295330796 +- 0.15186064004517466\n",
"Panoptic Quality IoU (pq): 0.719252280051296\n",
"Segmentation Quality Dsc (sq_dsc): 0.900292616009954 +- 0.10253566174957332\n",
"Panoptic Quality Dsc (pq_dsc): 0.7775254410995057\n",
"Segmentation Quality Assd (sq_assd): 0.250331887879225 +- 0.07696680402317076\n",
"\n"
]
}
],
"source": [
"# print all results\n",
"print(result)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"result.pq=0.719252280051296\n",
"\n"
],
"text/plain": [
"result.\u001b[33mpq\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.719252280051296\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# get specific metric, e.g. pq\n",
"pprint(f\"{result.pq=}\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"results dict: \n",
"{\n",
" 'num_ref_instances': 22,\n",
" 'num_pred_instances': 22,\n",
" 'tp': 19,\n",
" 'fp': 3,\n",
" 'fn': 3,\n",
" 'rq': 0.8636363636363636,\n",
" 'global_bin_dsc': 0.9744370224078394,\n",
" 'global_bin_cldsc': 0.9637064011802574,\n",
" 'sq': 0.8328184295330796,\n",
" 'sq_std': 0.15186064004517466,\n",
" 'pq': 0.719252280051296,\n",
" 'sq_dsc': 0.900292616009954,\n",
" 'sq_dsc_std': 0.10253566174957332,\n",
" 'pq_dsc': 0.7775254410995057,\n",
" 'sq_assd': 0.250331887879225,\n",
" 'sq_assd_std': 0.07696680402317076\n",
"}\n",
"\n"
],
"text/plain": [
"results dict: \n",
"\u001b[1m{\u001b[0m\n",
" \u001b[32m'num_ref_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n",
" \u001b[32m'num_pred_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n",
" \u001b[32m'tp'\u001b[0m: \u001b[1;36m19\u001b[0m,\n",
" \u001b[32m'fp'\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
" \u001b[32m'fn'\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
" \u001b[32m'rq'\u001b[0m: \u001b[1;36m0.8636363636363636\u001b[0m,\n",
" \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9744370224078394\u001b[0m,\n",
" \u001b[32m'global_bin_cldsc'\u001b[0m: \u001b[1;36m0.9637064011802574\u001b[0m,\n",
" \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n",
" \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.15186064004517466\u001b[0m,\n",
" \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.719252280051296\u001b[0m,\n",
" \u001b[32m'sq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n",
" \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.10253566174957332\u001b[0m,\n",
" \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.7775254410995057\u001b[0m,\n",
" \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.250331887879225\u001b[0m,\n",
" \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# get dict for further processing, e.g. for pandas\n",
"pprint(\"results dict: \", result.to_dict())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}