{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background:#FFFFAA\">\n",
    "<h1> Step 3: Making a submission </h1> \n",
    "\n",
    "<h2> Unit testing </h2> \n",
    "\n",
    "It is <b><span style=\"color:red\">important that you test your submission files before submitting them</span></b>. All you have to do to make a submission is create or modify the <code>Sumbission</code> class the file <code>submission.py</code>  in the <code>starting_kit/example_submission/</code> directory, then run this test to make sure everything works fine. This is the actual program that will be run on the server to test your submission. <br>\n",
    "\n",
    "take note that on codalab, your local directory is <code>program/</code>. Then if you want to load the file <code>model.dump</code> run : <code>open(\"program/model.dump\")</code> even if the file is in at the root of your submission dir.\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import logging\n",
    "import warnings\n",
    "import json\n",
    "from sys import path\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Uncomment the next lines to auto-reload libraries (this causes some problem with pickles in Python 3)\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import grid2op\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Useful paths for the submission and for data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "this_dir = os.getcwd()\n",
    "above_dir, final_dir = os.path.split(this_dir)\n",
    "above_above_dir, secondfinal_dir = os.path.split(above_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the first step tries to determine from where the jupyter notebook was started, and adapt the path accordingly\n",
    "if final_dir == \"starting_kit\":\n",
    "    # path where \"submission.py\" should be\n",
    "    model_dir = 'agent_path'\n",
    "    \n",
    "    # location of some utilities script to mimic codalab behaviour\n",
    "    utility_dir = \"../l2rpn2019_utils\"\n",
    "    utility_dir = os.path.abspath(utility_dir)\n",
    "    \n",
    "    # training data set\n",
    "    # if you didn't we encourage you to download the training data\n",
    "    path_train_set = '../data/data_l2rpn_2019'\n",
    "    path_train_set = os.path.abspath(path_train_set)\n",
    "    \n",
    "    # path where the final results and temporary data will be located\n",
    "    output_dir = 'output_notebook_2DevelopAndRunLocally'\n",
    "    tmp_outdir = os.path.join(output_dir, \"tmp_results\")\n",
    "    if not os.path.exists(output_dir):\n",
    "        print(\"Creating path \\\"{}\\\" where the output of this notebook will be stored\".format(output_dir))\n",
    "        os.mkdir(output_dir)\n",
    "    if not os.path.exists(tmp_outdir):\n",
    "        print(\"Creating path \\\"{}\\\" where the temporary output will be stored\".format(tmp_outdir))\n",
    "        os.mkdir(tmp_outdir)\n",
    "else:\n",
    "    raise RuntimeError(\"For this notebook to work properly, you should not have modified the github clone \"\n",
    "                       \"where you found the l2rpn_2019 utilities, starting kit and other materials\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The command bellow simulate the execution of your code on the training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done and data saved in : \"/home/benjamin/Documents/grid2op_dev/l2rpn_2019/starting_kit/output_notebook_2DevelopAndRunLocally/tmp_results/saved_experiment\"\r\n"
     ]
    }
   ],
   "source": [
    "# codalab command line used on the competition server. So it is best to test it as it is, \n",
    "# rather than directly importing the related python method here and execute it. \n",
    "!$sys.executable $utility_dir/ingestion.py $path_train_set $tmp_outdir $utility_dir $model_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background:#FFFFAA\">\n",
    "Also test the scoring program:\n",
    "    </div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_dir: output_notebook_2DevelopAndRunLocally/tmp_results\r\n",
      "output_dir: output_notebook_2DevelopAndRunLocally\r\n",
      "Score for scenario 0001: 1595.0512709067282\r\n",
      "Score for scenario 0000: 1542.1342069852371\r\n",
      "Your submission is valid, you may proceed with the next steps\r\n"
     ]
    }
   ],
   "source": [
    "!$sys.executable $utility_dir/evaluate.py $tmp_outdir $output_dir\n",
    "# print(\"watch : http:/view/\"+ scoring_output_dir +\"/scores.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"background:#FFFFAA\">\n",
    "    <h1> Preparing the submission </h1>\n",
    "\n",
    "Zip the contents of `sample_code_submission/` (without the directory), or download the challenge public_data and run the command in the previous cell, after replacing sample_data by public_data.\n",
    "Then zip the contents of `sample_result_submission/` (without the directory).\n",
    "<b><span style=\"color:red\">Do NOT zip the data with your submissions</span></b>."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime \n",
    "import zipfile\n",
    "\n",
    "def zipdir(path, ziph):\n",
    "    # ziph is zipfile handle\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file in files:\n",
    "            ziph.write(os.path.join(root, file))\n",
    "\n",
    "the_date = datetime.datetime.now().strftime(\"%y%m%d_%H%M\")\n",
    "sample_code_submission = 'sample_code_submission_' + the_date + '.zip' \n",
    "\n",
    "with zipfile.ZipFile(sample_code_submission, 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    zipdir(model_dir,zipf)\n",
    "print(\"Submit this file:\\n\\t- {}\\n\\t- Located at{}\".format(sample_code_submission, os.path.abspath(\".\")))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}