{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run run-validation-experiment.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = {'MIMIC' : \"~/cohorts/hs_mimic.csv\", 'SINAI' : \"~/cohorts/hs_sinai_preprocessed.csv\"}\n",
    "validation_path = './experiments/validation_mimic_on_sinai.d'\n",
    "derivation_path = './experiments/mimic_with_pipelines.d'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' define options to run experiments on '''\n",
    "options = {\n",
    "    'target': [\"AKI\"],\n",
    "    'cohort': ['SINAI'],\n",
    "    'bootstrap': ['yes'],\n",
    "    'n_iterations': [100], # number of iterations necessary\n",
    "    'n_ratio': [0.5], #how much of the data should we use for bootstrapping\n",
    "}\n",
    "\n",
    "''' retrieve previous experiments as the case may be '''\n",
    "derivation_experiments = unpickle(derivation_path) or {}\n",
    "validation_experiments = unpickle(validation_path) or {}\n",
    "n_validation_experiments = 0\n",
    "\n",
    "#filter validation experiments if needed\n",
    "\n",
    "with Timer() as t:    \n",
    "    ''' iterate over different options '''\n",
    "    for combination in product(*options.values()):\n",
    "    \n",
    "        ''' for every experiment saved on the pipeline '''\n",
    "        for derivation_experiment in derivation_experiments.values():\n",
    "\n",
    "            ''' initialize parameters '''\n",
    "            exp_id = str(uuid.uuid1())\n",
    "            params = dict(zip(options.keys(), combination))                       \n",
    "\n",
    "            ''' load the data '''\n",
    "            test = Load().execute(filename=filenames[params['cohort']])  \n",
    "                       \n",
    "            params['model_features'] = derivation_experiment['parameters']['model_features']\n",
    "            params['algorithm'] = derivation_experiment['parameters']['algorithm']\n",
    "            \n",
    "            print(f\"Running experiment with following parameters: {params}\")\n",
    "            \n",
    "            ''' evaluate on complete data using complete cohort and existing pipeline '''\n",
    "            validation_experiment = defaultdict(lambda: {})\n",
    "            validation_experiment = run_validation_experiment(params, test, **derivation_experiment['pipeline'])        \n",
    "                        \n",
    "            bs_performance = defaultdict(lambda:{})\n",
    "            bs_experiments = []\n",
    "            if params.get('bootstrap') == 'yes':           \n",
    "                                \n",
    "                metrics = ['precision', 'recall', 'f1-score', 'auc', 'dor']\n",
    "                n_iterations = params['n_iterations']\n",
    "                n_size = int(len(test) * params['n_ratio'])\n",
    "                \n",
    "                bs_exps = []\n",
    "                for i in range(n_iterations):\n",
    "                    print(f\"Validating bootstrapped sample #{i+1}\")\n",
    "                    bs_test = resample(test, n_samples=n_size)                    \n",
    "                    bs_experiments.append(run_validation_experiment(params, bs_test, **derivation_experiment['pipeline']))             \n",
    "                \n",
    "                for metric in metrics:\n",
    "                    measurements = [exp['performance']['discrimination'][metric] for exp in bs_experiments]\n",
    "                    bs_performance[metric]['mean'] = np.mean(measurements)\n",
    "                    bs_performance[metric]['std'] = np.std(measurements)\n",
    "                    bs_performance[metric]['ci'] = np.std(measurements) * 2 #95% CI                    \n",
    "            \n",
    "            ''' save everything '''\n",
    "            #validation_experiment['derivation_exp_id'] = derivation_experiment['exp_id']\n",
    "            validation_experiment['parameters'] = params\n",
    "            validation_experiment['performance']['bootstrap'] = bs_performance\n",
    "            validation_experiment['exp_id'] = exp_id\n",
    "            \n",
    "            validation_experiments[exp_id] = validation_experiment\n",
    "\n",
    "            n_validation_experiments += 1\n",
    "                            \n",
    "\n",
    "print(f'Running {n_validation_experiments} validation experiments took {t.interval:.03f} sec.')\n",
    "\n",
    "print(validation_experiments)\n",
    "\n",
    "''' store everything '''\n",
    "if pickle(validation_experiments, validation_path): print('Successfully saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}