{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e6c1a344-fdad-47b3-bd20-f1f90ffff2c9",
   "metadata": {},
   "source": [
    "https://mmdetection.readthedocs.io/en/latest/tutorials/test_results_submission.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "880b2f8f-8d5a-4d34-8cdd-6cd95495d635",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install git+https://github.com/MiXaiLL76/faster_coco_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9662b48e-ebd2-4cb3-a13c-1298d989937c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -pv data/coco/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac7b6b1f-4059-43f6-a6d5-64363fb5d289",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -P data/coco/ http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n",
    "!wget -P data/coco/ http://images.cocodataset.org/zips/val2017.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbcc1e4f-069c-4ad2-9c26-864bf111016a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!unzip data/coco/annotations_trainval2017.zip -d data/coco/\n",
    "!unzip data/coco/val2017.zip -d data/coco/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a1403ee-eff8-4e80-9e9a-69ba74d736f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf data/coco/*.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54ea5c5a-d294-45b2-a858-560050d9ecf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "yolo3_model_path = \"https://download.openmmlab.com/mmdetection/v2.0/yolo/yolov3_d53_320_273e_coco/yolov3_d53_320_273e_coco-421362b6.pth\"\n",
    "!wget -P model {yolo3_model_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e67a6ecb-d868-408e-8e52-661bd25df496",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cfg_path='configs/yolo/yolov3_d53_320_273e_coco.py'\n"
     ]
    }
   ],
   "source": [
    "import os.path as osp\n",
    "\n",
    "_BASE_CONFIG_DIR = \"configs/\"\n",
    "CONFIG_FILE     = \"yolo/yolov3_d53_320_273e_coco.py\"\n",
    "CHECKPOINT_FILE = \"model/yolov3_d53_320_273e_coco-421362b6.pth\"\n",
    "WORK_DIR = \".\"\n",
    "use_cpu = False\n",
    "\n",
    "cfg_path = osp.join(_BASE_CONFIG_DIR, CONFIG_FILE)\n",
    "print(f\"{cfg_path=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37d2b3c-72bb-4586-b379-2f1281978f05",
   "metadata": {},
   "outputs": [],
   "source": [
    "_dop = \"\"\n",
    "if use_cpu:\n",
    "    _dop += f\" --gpu-id -1 \"\n",
    "\n",
    "!python3 test.py \\\n",
    "    {cfg_path} \\\n",
    "    {CHECKPOINT_FILE} \\\n",
    "    --format-only {_dop}\\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json \\\n",
    "    data.test.img_prefix=data/coco/val2017 \\\n",
    "    --out yolo_result.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4eca90de-fd3d-4dc7-b3f4-ada1300d0b90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.28s)\n",
      "creating index...\n",
      "index created!\n",
      "Data uploaded for 1.057 sec.\n",
      "\n",
      "Evaluating bbox...\n",
      "Loading and preparing results...\n",
      "DONE (t=0.47s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=16.69s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=3.02s).\n",
      "\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.279\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=1000 ] = 0.491\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=1000 ] = 0.283\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.105\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.301\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.438\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.395\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=300 ] = 0.395\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=1000 ] = 0.395\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.185\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.423\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.574\n",
      "\n",
      "OrderedDict([('bbox_mAP', 0.279), ('bbox_mAP_50', 0.491), ('bbox_mAP_75', 0.283), ('bbox_mAP_s', 0.105), ('bbox_mAP_m', 0.301), ('bbox_mAP_l', 0.438), ('bbox_mAP_copypaste', '0.279 0.491 0.283 0.105 0.301 0.438')])\n",
      "Data validate for 22.854 sec.\n",
      "CPU times: user 160 ms, sys: 22.4 ms, total: 183 ms\n",
      "Wall time: 26 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "!python3 eval_metric.py {cfg_path} yolo_result.pkl \\\n",
    "    --eval bbox \\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8bda9c97-ecce-4ae2-ab1d-c36680fb3a70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.28s)\n",
      "creating index...\n",
      "index created!\n",
      "Data uploaded for 1.065 sec.\n",
      "\n",
      "Evaluating bbox...\n",
      "Loading and preparing results...\n",
      "DONE (t=0.47s)\n",
      "creating index...\n",
      "index created!\n",
      "\n",
      "\n",
      "OrderedDict([('bbox_mAP', 0.279), ('bbox_mAP_50', 0.491), ('bbox_mAP_75', 0.283), ('bbox_mAP_s', 0.105), ('bbox_mAP_m', 0.301), ('bbox_mAP_l', 0.438), ('bbox_mAP_copypaste', '0.279 0.491 0.283 0.105 0.301 0.438')])\n",
      "Data validate for 8.714 sec.\n",
      "CPU times: user 91.8 ms, sys: 5.82 ms, total: 97.6 ms\n",
      "Wall time: 11.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "!python3 eval_metric.py {cfg_path} yolo_result.pkl \\\n",
    "    --eval bbox \\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json data.test.type='FasterCocoDataset'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6ce86d4-f61e-4c18-821c-676d66a10d08",
   "metadata": {},
   "outputs": [],
   "source": [
    "yoloact_model_path = \"https://download.openmmlab.com/mmdetection/v2.0/yolact/yolact_r50_1x8_coco/yolact_r50_1x8_coco_20200908-f38d58df.pth\"\n",
    "!wget -P model {yoloact_model_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5685b448-6079-4f0a-a781-a3260b59e579",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cfg_path='configs/yolact/yolact_r50_1x8_coco.py'\n"
     ]
    }
   ],
   "source": [
    "CONFIG_FILE = \"yolact/yolact_r50_1x8_coco.py\"\n",
    "CHECKPOINT_FILE = \"model/yolact_r50_1x8_coco_20200908-f38d58df.pth\"\n",
    "\n",
    "cfg_path = osp.join(_BASE_CONFIG_DIR, CONFIG_FILE)\n",
    "print(f\"{cfg_path=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0129cb16-65ac-4192-b933-9d38fc4d5aee",
   "metadata": {},
   "outputs": [],
   "source": [
    "_dop = \"\"\n",
    "if use_cpu:\n",
    "    _dop += f\" --gpu-id -1 \"\n",
    "\n",
    "!python3 test.py \\\n",
    "    {cfg_path} \\\n",
    "    {CHECKPOINT_FILE} \\\n",
    "    --format-only {_dop}\\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json \\\n",
    "    data.test.img_prefix=data/coco/val2017 \\\n",
    "    --out yoloact_result.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "91c15c82-b82a-46c2-8171-f517ab677529",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.28s)\n",
      "creating index...\n",
      "index created!\n",
      "Data uploaded for 2.235 sec.\n",
      "\n",
      "Evaluating segm...\n",
      "/opt/conda/lib/python3.9/site-packages/mmdet/datasets/coco.py:470: UserWarning: The key \"bbox\" is deleted for more accurate mask AP of small/medium/large instances since v2.12.0. This does not change the overall mAP calculation.\n",
      "  warnings.warn(\n",
      "Loading and preparing results...\n",
      "DONE (t=1.21s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=22.02s).\n",
      "Accumulating evaluation results...\n",
      "/opt/conda/lib/python3.9/site-packages/pycocotools/cocoeval.py:378: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)\n",
      "DONE (t=3.49s).\n",
      "\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.290\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=1000 ] = 0.486\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=1000 ] = 0.296\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.100\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.315\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.465\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.392\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=300 ] = 0.392\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=1000 ] = 0.392\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=1000 ] = 0.176\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=1000 ] = 0.439\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=1000 ] = 0.568\n",
      "\n",
      "OrderedDict([('segm_mAP', 0.29), ('segm_mAP_50', 0.486), ('segm_mAP_75', 0.296), ('segm_mAP_s', 0.1), ('segm_mAP_m', 0.315), ('segm_mAP_l', 0.465), ('segm_mAP_copypaste', '0.290 0.486 0.296 0.100 0.315 0.465')])\n",
      "Data validate for 35.356 sec.\n",
      "CPU times: user 235 ms, sys: 22 ms, total: 257 ms\n",
      "Wall time: 39.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "!python3 eval_metric.py {cfg_path} yoloact_result.pkl \\\n",
    "    --eval segm \\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "312e736f-7108-4788-a47f-8b9bba26e10c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.28s)\n",
      "creating index...\n",
      "index created!\n",
      "Data uploaded for 2.305 sec.\n",
      "\n",
      "Evaluating segm...\n",
      "/home/rdl/storage/eval/coco_fast.py:105: UserWarning: The key \"bbox\" is deleted for more accurate mask AP of small/medium/large instances since v2.12.0. This does not change the overall mAP calculation.\n",
      "  warnings.warn(\n",
      "Loading and preparing results...\n",
      "DONE (t=1.25s)\n",
      "creating index...\n",
      "index created!\n",
      "\n",
      "\n",
      "OrderedDict([('segm_mAP', 0.29), ('segm_mAP_50', 0.486), ('segm_mAP_75', 0.296), ('segm_mAP_s', 0.1), ('segm_mAP_m', 0.315), ('segm_mAP_l', 0.465), ('segm_mAP_copypaste', '0.290 0.486 0.296 0.100 0.315 0.465')])\n",
      "Data validate for 18.403 sec.\n",
      "CPU times: user 103 ms, sys: 52.8 ms, total: 156 ms\n",
      "Wall time: 22.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "!python3 eval_metric.py {cfg_path} yoloact_result.pkl \\\n",
    "    --eval segm \\\n",
    "    --cfg-options data.test.ann_file=data/coco/annotations/instances_val2017.json data.test.type='FasterCocoDataset'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e85b179-87cc-4c7d-999a-a501ff9bb1d2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}