{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################################################################\n",
    "# set the parameters and press Ctrl-ENTER\n",
    "\n",
    "spacecraft = 'A'\n",
    "start = \"2016-01-01\" # included\n",
    "end = \"2016-02-01\" # not included\n",
    "#sampling = \"PT60S\" # set to None to use the original sampling\n",
    "sampling = None\n",
    "models = [\n",
    "    \"CHAOS-6-Core\",\n",
    "    \"CHAOS-6-Static\",\n",
    "    \"CHAOS-6-MMA-Primary\",\n",
    "    \"CHAOS-6-MMA-Secondary\"\n",
    "]\n",
    "variables=[\n",
    "    \"B_NEC\", \n",
    "    \"B_NEC_CHAOS-6-Core\",\n",
    "    \"B_NEC_CHAOS-6-Static\",\n",
    "    \"B_NEC_CHAOS-6-MMA-Primary\",\n",
    "    \"B_NEC_CHAOS-6-MMA-Secondary\",\n",
    "    'Flags_B', 'Flags_q', 'Flags_Platform', \n",
    "    'Kp', 'Dst', 'F10_INDEX', 'OrbitNumber', 'QDLat', 'QDLon', 'QDBasis', 'MLT',\n",
    "]\n",
    "\n",
    "output_filename_template = \"VIRES_MAG{spacecraft}_LR_1B_{start}_{end}.cdf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Can't download new leap second table\n",
      "Can't find leap second table.  Using one built into code.\n",
      "Last leap second in built in table is on Jan 01 2017. \n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from os import remove, rename\n",
    "from os.path import exists\n",
    "from viresclient._wps.wps_vires import ViresWPS10Service\n",
    "from viresclient._wps.environment import JINJA2_ENVIRONMENT\n",
    "from viresclient._wps.time_util import parse_datetime, parse_duration, encode_duration, timedelta, datetime\n",
    "from viresclient._client import ProgressBarProcessing, ProgressBarDownloading, ClientRequest\n",
    "\n",
    "wps_proxy = ViresWPS10Service(\n",
    "    url=\"https://staging.viresdisc.vires.services/openows\"\n",
    ")\n",
    "\n",
    "request_template = JINJA2_ENVIRONMENT.get_template(\n",
    "    \"vires_fetch_filtered_data_async.xml\"\n",
    ")\n",
    "\n",
    "spcectraft_to_collections_mapping = {\n",
    "    \"A\": \"SW_OPER_MAGA_LR_1B\",\n",
    "    \"B\": \"SW_OPER_MAGB_LR_1B\",\n",
    "    \"C\": \"SW_OPER_MAGC_LR_1B\",\n",
    "}\n",
    "\n",
    "def chop_time_range_by_days(start_time, end_time):\n",
    "    year, month, day = start_time.year, start_time.month, start_time.day\n",
    "\n",
    "    chunk_offset= datetime(year, month, day)\n",
    "    chunk_duration = timedelta(days=1, microseconds=-1)\n",
    "    chunk_step = timedelta(days=1)\n",
    "\n",
    "    while chunk_offset < end_time:\n",
    "        yield (chunk_offset, chunk_offset + chunk_duration)\n",
    "        chunk_offset += chunk_step\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2016-01-01 00:00:00\n",
      "2016-02-01 00:00:00\n",
      "2016-01-01 00:00:00 2016-01-01 23:59:59.999999\n",
      "2016-01-02 00:00:00 2016-01-02 23:59:59.999999\n",
      "2016-01-03 00:00:00 2016-01-03 23:59:59.999999\n",
      "2016-01-04 00:00:00 2016-01-04 23:59:59.999999\n",
      "2016-01-05 00:00:00 2016-01-05 23:59:59.999999\n",
      "2016-01-06 00:00:00 2016-01-06 23:59:59.999999\n",
      "2016-01-07 00:00:00 2016-01-07 23:59:59.999999\n",
      "2016-01-08 00:00:00 2016-01-08 23:59:59.999999\n",
      "2016-01-09 00:00:00 2016-01-09 23:59:59.999999\n",
      "2016-01-10 00:00:00 2016-01-10 23:59:59.999999\n",
      "2016-01-11 00:00:00 2016-01-11 23:59:59.999999\n",
      "2016-01-12 00:00:00 2016-01-12 23:59:59.999999\n",
      "2016-01-13 00:00:00 2016-01-13 23:59:59.999999\n",
      "2016-01-14 00:00:00 2016-01-14 23:59:59.999999\n",
      "2016-01-15 00:00:00 2016-01-15 23:59:59.999999\n",
      "2016-01-16 00:00:00 2016-01-16 23:59:59.999999\n",
      "2016-01-17 00:00:00 2016-01-17 23:59:59.999999\n",
      "2016-01-18 00:00:00 2016-01-18 23:59:59.999999\n",
      "2016-01-19 00:00:00 2016-01-19 23:59:59.999999\n",
      "2016-01-20 00:00:00 2016-01-20 23:59:59.999999\n",
      "2016-01-21 00:00:00 2016-01-21 23:59:59.999999\n",
      "2016-01-22 00:00:00 2016-01-22 23:59:59.999999\n",
      "2016-01-23 00:00:00 2016-01-23 23:59:59.999999\n",
      "2016-01-24 00:00:00 2016-01-24 23:59:59.999999\n",
      "2016-01-25 00:00:00 2016-01-25 23:59:59.999999\n",
      "2016-01-26 00:00:00 2016-01-26 23:59:59.999999\n",
      "2016-01-27 00:00:00 2016-01-27 23:59:59.999999\n",
      "2016-01-28 00:00:00 2016-01-28 23:59:59.999999\n",
      "2016-01-29 00:00:00 2016-01-29 23:59:59.999999\n",
      "2016-01-30 00:00:00 2016-01-30 23:59:59.999999\n",
      "2016-01-31 00:00:00 2016-01-31 23:59:59.999999\n",
      "request summary:\n",
      "\toutput_filenames:\n",
      "\t\tVIRES_MAGA_LR_1B_20160101-000000_20160101-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160102-000000_20160102-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160103-000000_20160103-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160104-000000_20160104-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160105-000000_20160105-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160106-000000_20160106-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160107-000000_20160107-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160108-000000_20160108-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160109-000000_20160109-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160110-000000_20160110-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160111-000000_20160111-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160112-000000_20160112-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160113-000000_20160113-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160114-000000_20160114-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160115-000000_20160115-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160116-000000_20160116-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160117-000000_20160117-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160118-000000_20160118-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160119-000000_20160119-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160120-000000_20160120-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160121-000000_20160121-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160122-000000_20160122-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160123-000000_20160123-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160124-000000_20160124-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160125-000000_20160125-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160126-000000_20160126-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160127-000000_20160127-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160128-000000_20160128-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160129-000000_20160129-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160130-000000_20160130-235959.cdf\n",
      "\t\tVIRES_MAGA_LR_1B_20160131-000000_20160131-235959.cdf\n",
      "\tstart time: 2016-01-01T00:00:00Z\n",
      "\tend time:   2016-02-01T00:00:00Z\n",
      "\tsampling:   original data sampling\n",
      "\tcollections:\n",
      "\tA: SW_OPER_MAGA_LR_1B\n",
      "\tmodels:\n",
      "\t\tCHAOS-6-Core\n",
      "\t\tCHAOS-6-Static\n",
      "\t\tCHAOS-6-MMA-Primary\n",
      "\t\tCHAOS-6-MMA-Secondary\n",
      "\tvariables:\n",
      "\t\tB_NEC\n",
      "\t\tB_NEC_CHAOS-6-Core\n",
      "\t\tB_NEC_CHAOS-6-Static\n",
      "\t\tB_NEC_CHAOS-6-MMA-Primary\n",
      "\t\tB_NEC_CHAOS-6-MMA-Secondary\n",
      "\t\tFlags_B\n",
      "\t\tFlags_q\n",
      "\t\tFlags_Platform\n",
      "\t\tKp\n",
      "\t\tDst\n",
      "\t\tF10_INDEX\n",
      "\t\tOrbitNumber\n",
      "\t\tQDLat\n",
      "\t\tQDLon\n",
      "\t\tQDBasis\n",
      "\t\tMLT\n",
      "\n",
      "processing VIRES_MAGA_LR_1B_20160101-000000_20160101-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160102-000000_20160102-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160103-000000_20160103-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160104-000000_20160104-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160105-000000_20160105-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160106-000000_20160106-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160107-000000_20160107-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160108-000000_20160108-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:54, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160109-000000_20160109-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:54, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160110-000000_20160110-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160111-000000_20160111-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160112-000000_20160112-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160113-000000_20160113-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160114-000000_20160114-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160115-000000_20160115-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160116-000000_20160116-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160117-000000_20160117-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160118-000000_20160118-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160119-000000_20160119-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160120-000000_20160120-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:54, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160121-000000_20160121-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:54, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160122-000000_20160122-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160123-000000_20160123-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160124-000000_20160124-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160125-000000_20160125-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160126-000000_20160126-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160127-000000_20160127-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160128-000000_20160128-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160129-000000_20160129-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160130-000000_20160130-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing VIRES_MAGA_LR_1B_20160131-000000_20160131-235959.cdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing:  100%|██████████|  [ Elapsed: 00:53, Remaining: 00:00 ]\n",
      "      Downloading: 100%|██████████|  [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)\n"
     ]
    }
   ],
   "source": [
    "start_time = parse_datetime(start)\n",
    "end_time = parse_datetime(end)\n",
    "\n",
    "# split time into daily chunks\n",
    "time_windows = list(chop_time_range_by_days(start_time, end_time))\n",
    "output_filenames = [\n",
    "    output_filename_template.format(\n",
    "        spacecraft=spacecraft,\n",
    "        start=chunk_start_time.strftime(\"%Y%m%d-%H%M%S\"),\n",
    "        end=chunk_end_time.strftime(\"%Y%m%d-%H%M%S\"),\n",
    "    ) for chunk_start_time, chunk_end_time in time_windows\n",
    "]\n",
    "\n",
    "# validate sampling step\n",
    "if sampling:\n",
    "    parse_duration(sampling)\n",
    "    \n",
    "# prepare collection identifiers\n",
    "collections = {\n",
    "    spacecraft: [spcectraft_to_collections_mapping[spacecraft]]\n",
    "}\n",
    "\n",
    "# print request summary\n",
    "print(\"request summary:\")\n",
    "print(\"\\toutput_filenames:\")\n",
    "for filename in output_filenames:\n",
    "    print(\"\\t\\t%s\" % filename)\n",
    "print(\"\\tstart time: %sZ\" % start_time.isoformat())\n",
    "print(\"\\tend time:   %sZ\" % end_time.isoformat())\n",
    "print(\"\\tsampling:   %s\" % (sampling or \"original data sampling\"))\n",
    "print(\"\\tcollections:\")\n",
    "for spacecraft, colls in collections.items():\n",
    "    print(\"\\t%s: %s\" % (spacecraft, \", \".join(colls)))\n",
    "print(\"\\tmodels:\")\n",
    "for model in models:\n",
    "    print(\"\\t\\t%s\" % model)\n",
    "print(\"\\tvariables:\")\n",
    "for variable in variables:\n",
    "    print(\"\\t\\t%s\" % variable)\n",
    "print()\n",
    "\n",
    "# download data by chunks\n",
    "for (chunk_start_time, chunk_end_time), output_filename in zip(time_windows, output_filenames):\n",
    "    print(\"processing %s\" % output_filename)\n",
    "    sys.stdout.flush()\n",
    "\n",
    "    # XML WPS request\n",
    "    request = request_template.render(\n",
    "        begin_time=chunk_start_time,\n",
    "        end_time=chunk_end_time,\n",
    "        sampling_step=sampling,\n",
    "        variables=variables,\n",
    "        collection_ids=collections,\n",
    "        model_ids=models,\n",
    "        response_type=\"application/x-cdf\",\n",
    "    )\n",
    "    \n",
    "    # download the data\n",
    "    if not exists(output_filename):\n",
    "        tmp_filename = output_filename + \".tmp\"\n",
    "        try:\n",
    "            with open(tmp_filename, \"wb\") as fout:\n",
    "                response_handler=ClientRequest._response_handler(fout)\n",
    "                with ProgressBarProcessing() as progress_bar:\n",
    "                    wps_proxy.retrieve_async(\n",
    "                        request.encode('UTF-8'),\n",
    "                        handler=response_handler,\n",
    "                        status_handler=progress_bar.update\n",
    "                    )\n",
    "        except Exception as error:\n",
    "            if exists(tmp_filename):\n",
    "                remove(tmp_filename)\n",
    "            print(\"ERROR: %s faled: %s\" % (output_filename, error), file=sys.stderr)\n",
    "        else:\n",
    "            if exists(tmp_filename):\n",
    "                rename(tmp_filename, output_filename)\n",
    "    else:\n",
    "        print(\"%s exists, skipped ... \" % output_filename)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (conda)",
   "language": "python",
   "name": "python_3.7_conda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}