{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gemini GSAOI reduction using DRAGONS Python API\n",
    "***\n",
    "## Public archival data from gsaoiimg_tutorial - GS-2017A-Q-29 (NGC5128)\n",
    "#### adapted from https://gsaoiimg-drtutorial.readthedocs.io/en/v2.1.1/03_api_reduction.html\n",
    "#### don't forget to `conda install -n dragons nb_conda_kernels ipykernel` to run this notebook on the DRAGONS env\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Importing Python Libraries** (you'll probably have to install the `wget`, `disco_stu`, and `ipympl` libraries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import glob\n",
    "import wget\n",
    "\n",
    "from gempy.adlibrary import dataselect\n",
    "from recipe_system import cal_service\n",
    "from recipe_system.reduction.coreReduce import Reduce\n",
    "from gempy.utils import logutils\n",
    "\n",
    "from astropy.io import fits\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.cbook as cbook\n",
    "from matplotlib.colors import LogNorm\n",
    "from matplotlib.colors import Normalize\n",
    "from matplotlib.colors import PowerNorm\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "from disco_stu import disco\n",
    "from disco_stu.lookups import general_parameters as disco_pars\n",
    "\n",
    "#%matplotlib widget\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (direct link to .tar file hosted at Gemini - 960 Megabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#wget.download(\"http://www.gemini.edu/sciops/data/software/datapkgs/gsaoiimg_tutorial_datapkg-v1.tar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (individual files - direct link to the Gemini Archive - 3.0 Gigabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Science \n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0095.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0096.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0097.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0098.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0099.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0100.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0101.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0102.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0103.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0104.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0105.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0106.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0107.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0108.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0109.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0110.fits\")\n",
    "\n",
    "# Flats \n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0030.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0031.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0032.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0033.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0034.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0035.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0036.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0037.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0038.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0039.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0040.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0041.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0042.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0043.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0044.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0060.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0061.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0062.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0063.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0064.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0065.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0066.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0067.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0068.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0069.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0070.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0071.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0072.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0073.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170505S0074.fits\")\n",
    "\n",
    "# Standard star\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170504S0114.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170504S0115.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170504S0116.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20170504S0117.fits\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create and move data to raw/ directory** (uncomment first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!mkdir raw/\n",
    "#!mv S2017*.fits raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check header of one raw science image\n",
    "#\n",
    "tmp = fits.open(\"raw/S20170505S0095.fits\")\n",
    "tmp[0].header"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the DRAGONS logger**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logutils.config(file_name='gsaoi_data_reduction.log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the Calibration Service**\n",
    "\n",
    "First, check that you have already a `rsys.cfg` file inside the `~/.geminidr/`. It should contain:\n",
    "\n",
    "```[calibs]\n",
    "standalone = True\n",
    "database_dir = /path_to_this_folder/```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "caldb = cal_service.CalibrationService()\n",
    "caldb.config()\n",
    "caldb.init()\n",
    "\n",
    "cal_service.set_calservice()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create a list of all the FITS files in the directory**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_files = glob.glob('raw/S2017*[0-9].fits')\n",
    "all_files.sort()\n",
    "#all_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**List of flats, standard star, and science frames**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_of_flats_Ks = dataselect.select_data(\n",
    "     all_files,['FLAT'],[],\n",
    "     dataselect.expr_parser('filter_name==\"Kshort\"'))\n",
    "\n",
    "list_of_std_stars = dataselect.select_data(\n",
    "    all_files,[],[],\n",
    "    dataselect.expr_parser('observation_class==\"partnerCal\"'))\n",
    "\n",
    "list_of_science_images = dataselect.select_data(\n",
    "    all_files,[],[],\n",
    "    dataselect.expr_parser('(observation_class==\"science\" and exposure_time==60.)'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Master Flat Field and add to the calibration database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_flats = Reduce()\n",
    "reduce_flats.files.extend(list_of_flats_Ks)\n",
    "reduce_flats.runr()\n",
    "\n",
    "caldb.add_cal(reduce_flats.output_filenames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reduce Standard Star**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_std = Reduce()\n",
    "reduce_std.files.extend(list_of_std_stars)\n",
    "reduce_std.runr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reduce Science Images**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_target = Reduce()\n",
    "reduce_target.files.extend(list_of_science_images)\n",
    "reduce_target.uparms.append(('skyCorrect:offset_sky', False))\n",
    "reduce_target.runr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Stack Sky-subtracted Images** (using `disco_stu` on the command line and not the API)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!disco `dataselect *_skySubtracted.fits --expr='observation_class==\"science\"'`\n",
    "#!disco `dataselect *_skySubtracted.fits --expr='observation_class==\"partnerCal\"'`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Display the Stacked Image**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_file = \"disco_stack_science.fits\"\n",
    "hdu_list = fits.open(image_file)\n",
    "hdu_list.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_data = fits.getdata(image_file, ext=1)\n",
    "print(image_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (15,15))\n",
    "plt.imshow(image_data,cmap='bone',norm=Normalize(vmin=1, vmax=1000))\n",
    "#plt.xlim(900,2500)\n",
    "#plt.ylim(1600,600)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Optional: remove duplicate calibrations and remove raw data (uncomment lines before running)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf *_flatCorrected.fits *_skySubtracted*.fits\n",
    "#!rm -rf raw/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dragons] *",
   "language": "python",
   "name": "conda-env-dragons-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}