{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gemini Flamingos-2 reduction using DRAGONS Python API\n",
    "***\n",
    "## Public archival data from f2img_tutorial - GS-2013B-Q-15 (WISE J041358.14-475039.3)\n",
    "#### adapted from https://f2img-drtutorial.readthedocs.io/en/v2.1.0/03_api_reduction.html\n",
    "#### don't forget to `conda install -n dragons nb_conda_kernels ipykernel` to run this notebook on the DRAGONS env \n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Importing Python Libraries** (you'll probably have to install the `wget` and `ipympl` libraries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import glob\n",
    "import wget\n",
    "\n",
    "from gempy.adlibrary import dataselect\n",
    "from recipe_system import cal_service\n",
    "from recipe_system.reduction.coreReduce import Reduce\n",
    "from gempy.utils import logutils\n",
    "\n",
    "from astropy.io import fits\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.cbook as cbook\n",
    "from matplotlib.colors import LogNorm\n",
    "from matplotlib.colors import Normalize\n",
    "from matplotlib.colors import PowerNorm\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "#%matplotlib widget\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (direct link to .tar file hosted at Gemini - 125 Megabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment the line and run the cell\n",
    "#\n",
    "#wget.download(\"http://www.gemini.edu/sciops/data/software/datapkgs/f2img_tutorial_datapkg-v1.tar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Downloading the data** (individual files - direct link to the Gemini Archive - 577 Megabytes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment the lines and run the cell\n",
    "#\n",
    "# Science \n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0075.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0076.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0077.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0078.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0079.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0080.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0081.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0082.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0083.fits\")\n",
    "#\n",
    "# Darks (2 s, short darks for BPM)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0369.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0370.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0371.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0372.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0373.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0374.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0375.fits\")\n",
    "#\n",
    "# Darks (120 s, for science data)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0115.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0116.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0117.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0118.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0119.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131120S0120.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131121S0010.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131122S0012.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131122S0438.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131122S0439.fits\")\n",
    "#\n",
    "# Flats (20 s, Lamp On, Y-band)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131129S0320.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131129S0321.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131129S0322.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131129S0323.fits\")\n",
    "#\n",
    "# Flats (20 s, Lamp Off, Y-band)\n",
    "#\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1111.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1112.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1113.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1114.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1115.fits\")\n",
    "#wget.download(\"http://archive.gemini.edu/file/S20131126S1116.fits\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create and move data to raw/ directory** (uncomment first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!mkdir raw/\n",
    "#!mv S2013*.fits raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check header of one raw science image\n",
    "#\n",
    "tmp = fits.open(\"raw/S20131121S0075.fits\")\n",
    "tmp[0].header"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the DRAGONS logger**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logutils.config(file_name='f2_data_reduction.log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setting up the Calibration Service**\n",
    "\n",
    "First, check that you have already a `rsys.cfg` file inside the `~/.geminidr/`. It should contain:\n",
    "\n",
    "```[calibs]\n",
    "standalone = True\n",
    "database_dir = /path_to_this_folder/```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "caldb = cal_service.CalibrationService()\n",
    "caldb.config()\n",
    "caldb.init()\n",
    "\n",
    "cal_service.set_calservice()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create a list of all the FITS files in the directory**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_files = glob.glob('raw/S2013*[0-9].fits')\n",
    "all_files.sort()\n",
    "#all_files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**List of darks, flats, and science frames**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dark_files_120s = dataselect.select_data(\n",
    "    all_files,['F2', 'DARK', 'RAW'],[],\n",
    "    dataselect.expr_parser('exposure_time==120'))\n",
    "\n",
    "dark_files_2s = dataselect.select_data(\n",
    "    all_files,['F2', 'DARK', 'RAW'],[],\n",
    "    dataselect.expr_parser('exposure_time==2'))\n",
    "\n",
    "list_of_flats_Y = dataselect.select_data(\n",
    "     all_files,['FLAT'],[],\n",
    "     dataselect.expr_parser('filter_name==\"Y\"'))\n",
    "\n",
    "list_of_science_images = dataselect.select_data(\n",
    "    all_files,['F2'],[],\n",
    "    dataselect.expr_parser('(observation_class==\"science\" and filter_name==\"Y\")'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Master Dark and add to the calibration database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_darks = Reduce()\n",
    "reduce_darks.files.extend(dark_files_120s)\n",
    "reduce_darks.runr()\n",
    "\n",
    "caldb.add_cal(reduce_darks.output_filenames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Bad Pixel Mask**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_bpm = Reduce()\n",
    "reduce_bpm.files.extend(list_of_flats_Y)\n",
    "reduce_bpm.files.extend(dark_files_2s)\n",
    "reduce_bpm.recipename = 'makeProcessedBPM'\n",
    "reduce_bpm.runr()\n",
    "\n",
    "bpm_filename = reduce_bpm.output_filenames[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create Master Flat Field and add to the calibration database**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_flats = Reduce()\n",
    "reduce_flats.files.extend(list_of_flats_Y)\n",
    "reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]\n",
    "reduce_flats.runr()\n",
    "\n",
    "caldb.add_cal(reduce_flats.output_filenames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reduce Science Images**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_target = Reduce()\n",
    "reduce_target.files.extend(list_of_science_images)\n",
    "reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]\n",
    "reduce_target.runr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Display the Stacked Image**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_file = \"S20131121S0075_stack.fits\"\n",
    "hdu_list = fits.open(image_file)\n",
    "hdu_list.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_data = fits.getdata(image_file, ext=1)\n",
    "print(image_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (15,15))\n",
    "plt.imshow(image_data,cmap='bone',norm=Normalize(vmin=1, vmax=1000))\n",
    "#plt.xlim(900,2500)\n",
    "#plt.ylim(1600,600)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Optional: remove raw data (uncomment lines before running)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf raw/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dragons] *",
   "language": "python",
   "name": "conda-env-dragons-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}