{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Can't download new leap second table\n", "Can't find leap second table. Using one built into code.\n", "Last leap second in built in table is on Jan 01 2017. \n" ] } ], "source": [ "import sys\n", "from os import remove, rename\n", "from os.path import exists\n", "from viresclient._wps.wps_vires import ViresWPS10Service\n", "from viresclient._wps.environment import JINJA2_ENVIRONMENT\n", "from viresclient._wps.time_util import parse_datetime, parse_duration, encode_duration, timedelta, datetime\n", "from viresclient._client import ProgressBarProcessing, ProgressBarDownloading, ClientRequest\n", "\n", "wps_proxy = ViresWPS10Service(\n", " url=\"https://staging.viresdisc.vires.services/openows\"\n", ")\n", "\n", "request_template = JINJA2_ENVIRONMENT.get_template(\n", " \"vires_fetch_filtered_data_async.xml\"\n", ")\n", "\n", "spacecraft_to_collections_mapping = {\n", " \"A\": \"SW_OPER_MAGA_LR_1B\",\n", " \"B\": \"SW_OPER_MAGB_LR_1B\",\n", " \"C\": \"SW_OPER_MAGC_LR_1B\",\n", "}\n", "\n", "def chop_timerange_by_months(start_time, end_time):\n", " year, month = start_time.year, start_time.month\n", " window_end = start_time\n", " while window_end < end_time:\n", " next_year, next_month = year, month + 1\n", " if next_month > 12:\n", " next_year += 1\n", " next_month -= 12\n", " window_start = max(start_time, datetime(year, month, 1))\n", " window_end = min(end_time, datetime(next_year, next_month, 1) - timedelta(microseconds=1))\n", " yield (window_start, window_end)\n", " year, month = next_year, next_month\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "request summary:\n", "\toutput_filenames:\n", "\t\tCHAOS6_residuals_A_20160101-000000_20160101-235959.cdf\n", "\tstart time: 2016-01-01T00:00:00Z\n", "\tend time: 2016-01-01T23:59:59.999999Z\n", "\tsampling: original data sampling\n", "\tcollections:\n", "\tA: SW_OPER_MAGA_LR_1B\n", "\tmodels:\n", "\t\tCHAOS-6-Combined\n", "\tvariables:\n", "\t\tB_NEC_res_CHAOS-6-Combined\n", "\t\tB_NEC\n", "\t\tB_NEC_CHAOS-6-Combined\n", "\t\tF_res_CHAOS-6-Combined\n", "\t\tF\n", "\t\tF_CHAOS-6-Combined\n", "\t\tdF_AOCS\n", "\t\tdF_other\n", "\t\tF_error\n", "\t\tB_VFM\n", "\t\tdB_Sun\n", "\t\tdB_AOCS\n", "\t\tdB_other\n", "\t\tB_error\n", "\t\tq_NEC_CRF\n", "\t\tAtt_error\n", "\t\tFlags_F\n", "\t\tFlags_B\n", "\t\tFlags_q\n", "\t\tFlags_Platform\n", "\t\tASM_Freq_Dev\n", "\t\tKp\n", "\t\tDst\n", "\t\tF107\n", "\t\tOrbitNumber\n", "\t\tQDLat\n", "\t\tQDLon\n", "\t\tQDBasis\n", "\t\tMLT\n", "\t\tSunDeclination\n", "\t\tSunHourAngle\n", "\t\tSunRightAscension\n", "\t\tSunAzimuthAngle\n", "\t\tSunZenithAngle\n", "\t\tSunLongitude\n", "\n", "processing CHAOS6_residuals_A_20160101-000000_20160101-235959.cdf\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing: 100%|██████████| [ Elapsed: 00:01, Remaining: 00:00 ]\n", " Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (34.058MB)\n" ] } ], "source": [ "###################################################################################\n", "# set the parameters and press Ctrl-ENTER\n", "\n", "start = \"2016-01-01\" # included\n", "end = \"2016-01-02\" # not included\n", "#sampling = \"PT60S\" # set to None to use the original sampling\n", "sampling = None\n", "spacecraft = 'A' # A B C\n", "models = [\"CHAOS-6-Combined\"]\n", "variables=[\n", " \"B_NEC_res_CHAOS-6-Combined\", \"B_NEC\", \"B_NEC_CHAOS-6-Combined\",\n", " \"F_res_CHAOS-6-Combined\", \"F\", \"F_CHAOS-6-Combined\",\n", " 'dF_AOCS', 'dF_other', 'F_error', 'B_VFM', 'dB_Sun',\n", " 'dB_AOCS', 'dB_other', 'B_error', 'q_NEC_CRF', 'Att_error', 'Flags_F',\n", " 'Flags_B', 'Flags_q', 'Flags_Platform', 'ASM_Freq_Dev',\n", " 'Kp', 'Dst', 'F107', 'OrbitNumber', 'QDLat', 'QDLon', 'QDBasis', 'MLT',\n", " 'SunDeclination', 'SunHourAngle', 'SunRightAscension',\n", " 'SunAzimuthAngle', 'SunZenithAngle', 'SunLongitude',\n", "]\n", "\n", "output_filename_template = \"CHAOS6_residuals_{spacecraft}_{start}_{end}.cdf\"\n", "\n", "###################################################################################\n", "\n", "start_time = parse_datetime(start)\n", "end_time = max(start_time, parse_datetime(end) + timedelta(microseconds=-1))\n", "\n", "# split time into monthly chunks\n", "# note that VirES will not allow more than 50 days of 1s samped data\n", "\n", "time_windows = list(chop_timerange_by_months(start_time, end_time))\n", "output_filenames = [\n", " output_filename_template.format(\n", " spacecraft=spacecraft,\n", " start=chunk_start_time.strftime(\"%Y%m%d-%H%M%S\"),\n", " end=chunk_end_time.strftime(\"%Y%m%d-%H%M%S\"),\n", " ) for chunk_start_time, chunk_end_time in time_windows\n", "]\n", "\n", "# validate sampling step\n", "if sampling:\n", " parse_duration(sampling)\n", " \n", "# prepare collection identifiers\n", "collections = {\n", " spacecraft: [spacecraft_to_collections_mapping[spacecraft]]\n", "}\n", "\n", "# print request summary\n", "print(\"request summary:\")\n", "print(\"\\toutput_filenames:\")\n", "for filename in output_filenames:\n", " print(\"\\t\\t%s\" % filename)\n", "print(\"\\tstart time: %sZ\" % start_time.isoformat())\n", "print(\"\\tend time: %sZ\" % end_time.isoformat())\n", "print(\"\\tsampling: %s\" % (sampling or \"original data sampling\"))\n", "print(\"\\tcollections:\")\n", "for spacecraft, colls in collections.items():\n", " print(\"\\t%s: %s\" % (spacecraft, \", \".join(colls)))\n", "print(\"\\tmodels:\")\n", "for model in models:\n", " print(\"\\t\\t%s\" % model)\n", "print(\"\\tvariables:\")\n", "for variable in variables:\n", " print(\"\\t\\t%s\" % variable)\n", "print()\n", "\n", "# download data by chunks\n", "for (chunk_start_time, chunk_end_time), output_filename in zip(time_windows, output_filenames):\n", " print(\"processing %s\" % output_filename)\n", " sys.stdout.flush()\n", "\n", " # XML WPS request\n", " request = request_template.render(\n", " begin_time=chunk_start_time,\n", " end_time=chunk_end_time,\n", " sampling_step=sampling,\n", " variables=variables,\n", " collection_ids=collections,\n", " model_expressions=\",\".join(models),\n", " response_type=\"application/x-cdf\",\n", " )\n", " \n", " # download the data\n", " if not exists(output_filename):\n", " tmp_filename = output_filename + \".tmp\"\n", " try:\n", " with open(tmp_filename, \"wb\") as fout:\n", " response_handler=ClientRequest._response_handler(fout)\n", " with ProgressBarProcessing() as progress_bar:\n", " wps_proxy.retrieve_async(\n", " request.encode('UTF-8'),\n", " handler=response_handler,\n", " status_handler=progress_bar.update\n", " )\n", " except Exception as error:\n", " if exists(tmp_filename):\n", " remove(tmp_filename)\n", " print(\"ERROR: %s faled: %s\" % (output_filename, error), file=sys.stderr)\n", " else:\n", " if exists(tmp_filename):\n", " rename(tmp_filename, output_filename)\n", " else:\n", " print(\"%s exists, skipped ... \" % output_filename)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.7 (conda)", "language": "python", "name": "python_3.7_conda" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.0" } }, "nbformat": 4, "nbformat_minor": 2 }