{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gemini GNIRS reduction using DRAGONS Python API\n",
    "***\n",
    "## Public archival data from gnirsimg_tutorial - GN-2011B-Q-34 (GRB120116A)\n",
    "#### adapted from https://gnirsimg-drtutorial.readthedocs.io/en/v2.1.0/04_keyhole_api.html\n",
    "#### don't forget to `conda install -n dragons nb_conda_kernels ipykernel` to run this notebook on the DRAGONS env \n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Importing Python Libraries** (you'll probably have to install the `wget` and `ipympl` libraries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import glob\n",
    "import wget\n",
    "\n",
    "from gempy.adlibrary import dataselect\n",
    "from recipe_system import cal_service\n",
    "from recipe_system.reduction.coreReduce import Reduce\n",
    "from recipe_system.utils.reduce_utils import normalize_ucals\n",
    "from gempy.utils import logutils\n",
    "\n",
    "from astropy.io import fits\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.cbook as cbook\n",
    "from matplotlib.colors import LogNorm\n",
    "from matplotlib.colors import Normalize\n",
    "from matplotlib.colors import PowerNorm\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "#%matplotlib widget\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (direct link to .tar file hosted at Gemini - 35 Megabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment the line and run the cell\n",
    "#\n",
    "#wget.download(\"http://www.gemini.edu/sciops/data/software/datapkgs/gnirsimg_tutorial_datapkg-v1.tar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (individual files - direct link to the Gemini Archive - 184 Megabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment the lines and run the cell\n",
    "#\n",
    "# Science \n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0014.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0015.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0016.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0017.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0018.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0019.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0020.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0021.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0022.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0023.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0024.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0025.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0026.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0027.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0028.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0029.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0030.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0031.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0032.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0033.fits\")\n",
    "\n",
    "# Science darks\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0538.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0539.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0540.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0541.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0542.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0543.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0544.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0545.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0546.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120102S0547.fits\")\n",
    "\n",
    "# Flats (lamp-on)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0034.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0035.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0036.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0037.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0038.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0039.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0040.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0041.fits\")\n",
    "\n",
    "# Flats (lamp-off)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0042.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0043.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0044.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0045.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0046.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0047.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0048.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/N20120117S0049.fits\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create and move data to raw/ directory** (uncomment first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!mkdir raw/\n",
    "#!mv N2012*.fits raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check header of one raw science image\n",
    "#\n",
    "tmp = fits.open(\"raw/N20120117S0014.fits\")\n",
    "tmp[0].header"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the DRAGONS logger**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logutils.config(file_name='gnirs_data_reduction.log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the Calibration Service**\n",
    "\n",
    "First, check that you have already a `rsys.cfg` file inside the `~/.geminidr/`. It should contain:\n",
    "\n",
    "```[calibs]\n",
    "standalone = True\n",
    "database_dir = /path_to_this_folder/```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "caldb = cal_service.CalibrationService()\n",
    "caldb.config()\n",
    "caldb.init()\n",
    "\n",
    "cal_service.set_calservice()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create a list of all the FITS files in the directory**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_files = glob.glob('raw/N2012*[0-9].fits')\n",
    "all_files.sort()\n",
    "#all_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**List of darks, flats, standard star, and science frames**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "darks60 = dataselect.select_data(\n",
    "    all_files,\n",
    "    ['DARK'],\n",
    "    [],\n",
    "    dataselect.expr_parser('exposure_time==60')\n",
    ")\n",
    "\n",
    "flats = dataselect.select_data(all_files, ['FLAT'])\n",
    "\n",
    "target = dataselect.select_data(\n",
    "    all_files,\n",
    "    [],\n",
    "    [],\n",
    "    dataselect.expr_parser('object==\"GRB120116A\"')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Master Dark and add to the calibration database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_darks = Reduce()\n",
    "reduce_darks.files.extend(darks60)\n",
    "reduce_darks.runr()\n",
    "\n",
    "caldb.add_cal(reduce_darks.output_filenames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Master Flat Field and add to the calibration database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_flats = Reduce()\n",
    "reduce_flats.files.extend(flats)\n",
    "reduce_flats.runr()\n",
    "\n",
    "caldb.add_cal(reduce_flats.output_filenames[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!showpars raw/N20120117S0014.fits associateSky"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reduce Science Images**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mycalibrations = ['processed_dark:N20120102S0538_dark.fits']\n",
    "\n",
    "reduce_target = Reduce()\n",
    "reduce_target.files.extend(target)\n",
    "ucals_dict = normalize_ucals(reduce_target.files, mycalibrations)\n",
    "reduce_target.ucals = ucals_dict\n",
    "reduce_target.runr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Display the Stacked Image**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_file = \"N20120117S0014_stack.fits\"\n",
    "hdu_list = fits.open(image_file)\n",
    "hdu_list.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_data = fits.getdata(image_file, ext=1)\n",
    "print(image_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (15,15))\n",
    "plt.imshow(image_data,cmap='bone',norm=Normalize(vmin=1, vmax=500))\n",
    "#plt.xlim(900,2500)\n",
    "#plt.ylim(1600,600)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Optional: remove raw data (uncomment lines before running)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf raw/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dragons] *",
   "language": "python",
   "name": "conda-env-dragons-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}