{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%html\n",
""
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"import datetime\n",
"import pprint\n",
"import pyaurorax\n",
"\n",
"aurorax = pyaurorax.PyAuroraX()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Download data"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Name Provider Level DOI Details Short Description \n",
"================================================================================================================================================================================================\n",
"REGO_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Flatfield calibration data (IDL save format) \n",
"REGO_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers Rayleighs calibration data (IDL save format) \n",
"REGO_DAILY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (JPG format) \n",
"REGO_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PGM format) \n",
"REGO_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily keogram summary product (PNG format) \n",
"REGO_DAILY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (JPG format) \n",
"REGO_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PGM format) \n",
"REGO_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers daily montage summary product (PNG format) \n",
"REGO_HOURLY_KEOGRAM_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (JPG format) \n",
"REGO_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PGM format) \n",
"REGO_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly keogram summary product (PNG format) \n",
"REGO_HOURLY_MONTAGE_JPG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (JPG format) \n",
"REGO_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PGM format) \n",
"REGO_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers hourly montage summary product (PNG format) \n",
"REGO_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers 3-sec raw data \n",
"REGO_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/z7x6-5c42 REGO All Sky Imagers skymap data (IDL save format) \n",
"THEMIS_ASI_DAILY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (JPG format) \n",
"THEMIS_ASI_DAILY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers daily keogram summary product (PGM format) \n",
"THEMIS_ASI_DAILY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (JPG format) \n",
"THEMIS_ASI_DAILY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers daily montage summary product (PGM format) \n",
"THEMIS_ASI_HOURLY_AVERAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (JPG format) \n",
"THEMIS_ASI_HOURLY_AVERAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly average summary product (PGM format) \n",
"THEMIS_ASI_HOURLY_KEOGRAM_JPG UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (JPG format) \n",
"THEMIS_ASI_HOURLY_KEOGRAM_PGM UCalgary L2 None THEMIS All Sky Imagers hourly keogram summary product (PGM format) \n",
"THEMIS_ASI_HOURLY_MONTAGE_JPG UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (JPG format) \n",
"THEMIS_ASI_HOURLY_MONTAGE_PGM UCalgary L2 None THEMIS All Sky Imagers hourly montage summary product (PGM format) \n",
"THEMIS_ASI_RAW UCalgary L0 None THEMIS All Sky Imagers 3-sec raw data \n",
"THEMIS_ASI_RAW_ROW2 UCalgary L0 None THEMIS All Sky Imagers raw row2 data \n",
"THEMIS_ASI_RAW_WIDE UCalgary L0 None THEMIS All Sky Imagers raw wide data \n",
"THEMIS_ASI_SKYMAP_IDLSAV UCalgary L3 None THEMIS All Sky Imagers skymap data (IDL save format) \n",
"THEMIS_ASI_THUMB32 UCalgary L1A None THEMIS All Sky Imagers thumb32 data \n",
"THEMIS_ASI_VEC1024 UCalgary L1A None THEMIS All Sky Imagers vec1024 data \n",
"TREX_BLUE_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Flatfield calibration data (IDL save format)\n",
"TREX_BLUE_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers Rayleighs calibration data (IDL save format)\n",
"TREX_BLUE_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PGM format) \n",
"TREX_BLUE_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily keogram summary product (PNG format) \n",
"TREX_BLUE_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PGM format) \n",
"TREX_BLUE_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers daily montage summary product (PNG format) \n",
"TREX_BLUE_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PGM format) \n",
"TREX_BLUE_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly keogram summary product (PNG format) \n",
"TREX_BLUE_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PGM format) \n",
"TREX_BLUE_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers hourly montage summary product (PNG format) \n",
"TREX_BLUE_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers 3-sec raw data \n",
"TREX_BLUE_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/80pf-0p02 TREx Blueline All Sky Imagers skymap data (IDL save format) \n",
"TREX_NIR_CALIBRATION_FLATFIELD_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Flatfield calibration data (IDL save format) \n",
"TREX_NIR_CALIBRATION_RAYLEIGHS_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers Rayleighs calibration data (IDL save format) \n",
"TREX_NIR_DAILY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PGM format) \n",
"TREX_NIR_DAILY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily keogram summary product (PNG format) \n",
"TREX_NIR_DAILY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PGM format) \n",
"TREX_NIR_DAILY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers daily montage summary product (PNG format) \n",
"TREX_NIR_HOURLY_KEOGRAM_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PGM format) \n",
"TREX_NIR_HOURLY_KEOGRAM_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly keogram summary product (PNG format) \n",
"TREX_NIR_HOURLY_MONTAGE_PGM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PGM format) \n",
"TREX_NIR_HOURLY_MONTAGE_PNG UCalgary L2 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers hourly montage summary product (PNG format) \n",
"TREX_NIR_RAW UCalgary L0 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers 6-sec raw data \n",
"TREX_NIR_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/98w7-jp47 TREx NIR All Sky Imagers skymap data (IDL save format) \n",
"TREX_RGB_DAILY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily keogram summary product \n",
"TREX_RGB_DAILY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers daily montage summary product \n",
"TREX_RGB_HOURLY_KEOGRAM UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly keogram summary product \n",
"TREX_RGB_HOURLY_MONTAGE UCalgary L2 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers hourly montage summary product \n",
"TREX_RGB_RAW_BURST UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3Hz raw burst data \n",
"TREX_RGB_RAW_NOMINAL UCalgary L0 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers 3-sec raw data \n",
"TREX_RGB_SKYMAP_IDLSAV UCalgary L3 https://commons.datacite.org/doi.org/10.11575/4p8e-1k65 TREx RGB All Sky Imagers skymap data (IDL save format) \n"
]
}
],
"source": [
"# list available datasets\n",
"aurorax.data.list_datasets_in_table()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "97a70ae7468f47afaf531d64098ea611",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading THEMIS_ASI_RAW files: 0%| | 0.00/128M [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# download an hour of THEMIS ASI data\n",
"dataset_name = \"THEMIS_ASI_RAW\"\n",
"start_dt = datetime.datetime(2021, 11, 4, 9, 0)\n",
"end_dt = datetime.datetime(2021, 11, 4, 9, 59)\n",
"site_uid = \"atha\"\n",
"r = aurorax.data.ucalgary.download(dataset_name, start_dt, end_dt, site_uid=site_uid)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"FileDownloadResult(filenames=[WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0900_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0901_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0902_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0903_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0904_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0905_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0906_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0907_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0908_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0909_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0910_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0911_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0912_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0913_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0914_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0915_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0916_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0917_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0918_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0919_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0920_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0921_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0922_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0923_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0924_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0925_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0926_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0927_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0928_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0929_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0930_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0931_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0932_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0933_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0934_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0935_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0936_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0937_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0938_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0939_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0940_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0941_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0942_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0943_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0944_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0945_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0946_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0947_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0948_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0949_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0950_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0951_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0952_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0953_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0954_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0955_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0956_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0957_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0958_atha_themis02_full.pgm.gz'), WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0959_atha_themis02_full.pgm.gz')], count=60, total_bytes=127920163, output_root_path='C:\\\\Users\\\\darrenc\\\\pyaurorax_data/THEMIS_ASI_RAW', dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-sec raw data', provider='UCalgary', level='L0', doi_details='None', ...))\n",
"\n",
"{'count': 60,\n",
" 'dataset': Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-sec raw data', provider='UCalgary', level='L0', doi_details='None', ...),\n",
" 'filenames': [WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0900_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0901_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0902_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0903_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0904_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0905_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0906_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0907_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0908_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0909_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0910_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0911_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0912_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0913_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0914_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0915_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0916_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0917_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0918_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0919_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0920_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0921_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0922_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0923_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0924_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0925_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0926_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0927_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0928_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0929_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0930_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0931_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0932_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0933_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0934_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0935_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0936_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0937_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0938_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0939_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0940_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0941_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0942_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0943_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0944_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0945_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0946_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0947_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0948_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0949_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0950_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0951_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0952_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0953_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0954_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0955_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0956_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0957_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0958_atha_themis02_full.pgm.gz'),\n",
" WindowsPath('C:/Users/darrenc/pyaurorax_data/THEMIS_ASI_RAW/2021/11/04/atha_themis02/ut09/20211104_0959_atha_themis02_full.pgm.gz')],\n",
" 'output_root_path': 'C:\\\\Users\\\\darrenc\\\\pyaurorax_data/THEMIS_ASI_RAW',\n",
" 'total_bytes': 127920163}\n"
]
}
],
"source": [
"# view information about the downloaded data\n",
"print(r)\n",
"print()\n",
"\n",
"pprint.pprint(r.__dict__)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7422451387c34a1498fa637b571d4065",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading THEMIS_ASI_RAW files: 0%| | 0.00/136M [00:00, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# download a couple minutes of data from all THEMIS sites\n",
"dataset_name = \"THEMIS_ASI_RAW\"\n",
"start_dt = datetime.datetime(2021, 11, 4, 9, 0)\n",
"end_dt = datetime.datetime(2021, 11, 4, 9, 4)\n",
"_ = aurorax.data.ucalgary.download(dataset_name, start_dt, end_dt)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Read data\n",
"\n",
"The data reading routines take in a list of filenames on your computer. The list of files is returned in a `download()` call, but can also be created using `glob` or similar.\n",
"\n",
"There are two methods that can be used for reading data:\n",
"\n",
"1) using the generic method\n",
"2) using a specific dataset read function\n",
"\n",
"The generic method is the recommended way, but if more control is wanted, you can use the specific read functions directly. The generic method simply uses the dataset name to determine which specific read function to use.\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Data(data=array(dims=(256, 256, 1200), dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n",
"\n",
"Data:\n",
" data : array(dims=(256, 256, 1200), dtype=uint16)\n",
" timestamp : [1200 datetimes]\n",
" metadata : [1200 dictionaries]\n",
" problematic_files : []\n",
" calibrated_data : None\n",
" dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n"
]
}
],
"source": [
"# let's show the generic method first, since it is the easiest way\n",
"data = aurorax.data.ucalgary.read(r.dataset, r.filenames, n_parallel=2)\n",
"\n",
"print(data)\n",
"print()\n",
"\n",
"data.pretty_print()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Data(data=array(dims=(256, 256, 1200), dtype=uint16), timestamp=[1200 datetimes], metadata=[1200 dictionaries], problematic_files=[], calibrated_data=None, dataset=Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...))\n",
"\n",
"Data:\n",
" data : array(dims=(256, 256, 1200), dtype=uint16)\n",
" timestamp : [1200 datetimes]\n",
" metadata : [1200 dictionaries]\n",
" problematic_files : []\n",
" calibrated_data : None\n",
" dataset : Dataset(name=THEMIS_ASI_RAW, short_description='THEMIS All Sky Imagers 3-se...)\n"
]
}
],
"source": [
"# since we know we're reading in THEMIS raw data, we can also use the specific read routine. In most\n",
"# circumstances, this method isn't necessary to use.\n",
"data = aurorax.data.ucalgary.readers.read_themis(r.filenames, n_parallel=2, dataset=r.dataset)\n",
"\n",
"print(data)\n",
"print()\n",
"\n",
"data.pretty_print()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Managing downloaded data"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dataset name Size \n",
"=========================\n",
"THEMIS_ASI_RAW 253.6 MB\n",
"\n",
"Total size: 253.6 MB\n"
]
}
],
"source": [
"# to view the amount of data that is currently downloaded, do the following\n",
"aurorax.show_data_usage()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# alternatively, you can get the data usage information returned as a dictionary\n",
"data_usage_dict = aurorax.show_data_usage(return_dict=True)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"# to clean up all data we've downloaded, you can do the following.\n",
"# \n",
"# alternatively, you can manually delete data yourself\n",
"aurorax.purge_download_output_root_path()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "pyaurorax39",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.19"
}
},
"nbformat": 4,
"nbformat_minor": 2
}