{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Automatically download FESOM2 data from Zenodo based on catalog"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Imports and paths"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import intake\n",
"import os\n",
"from pathlib import Path\n",
"from esmvfc_cattools import download_zenodo_files_for_entry\n",
"import logging"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# parameters\n",
"data_path = Path(\"../esm_vfc_data/\").resolve()\n",
"catalog_file = Path(\"../catalogs/FESOM2_PI_MESH.yaml\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"ESM_VFC_DATA_DIR\"] = str(data_path)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Open catalog and download all data"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"cat = intake.open_catalog(str(catalog_file))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['FESOM2_sample', 'MESH_NOD2D', 'MESH_ELEM2D', 'MESH_AUX3D']"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(cat)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/a_ice.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/a_ice.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f120384213a94250a73011e766594958",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/Av.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/Av.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e8d5faf02e3a49438b8cf45c53ee481c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/Kv.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/Kv.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "549b14ed586d47de9e28fc734d973ae9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/m_ice.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/m_ice.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4ea9c56cd03c447c8da693d6fa145f7e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/MLD1.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/MLD1.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5bb69bd7edf244a3ac1d7271888fa147",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/salt.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/salt.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "72d27f6ba4904c21a11a9c4358ab073a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/ssh.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/ssh.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "52d4789f292a42b18be05fab74a60467",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/sst.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/sst.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "daf177b8b13f48b3996fc894c0730859",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/temp.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/temp.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "518b636804e541f7afab3f73258ec0cc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/u.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/u.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "50f65be5e2ae4c818b46c8c267df47e9",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/uice.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/uice.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c08b1738761f47aaac45b9e26b79cc84",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/unod.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/unod.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7ab87ba4e9fd45d3b2a3cf3292bdfd8f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/v.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/v.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "50481635469b481595c29a9f9a2db378",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/vice.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/vice.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4859cae9121a4c3598a172b734e7d741",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/vnod.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/vnod.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "46153c47dbbe4c79bc614b752fed4be0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/w.fesom.1948.nc to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/w.fesom.1948.nc\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "96c26b694bf44beaad0e40ea930c6125",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/pi.zip\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a5348225ec46409491686f72d6bf8126",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/pi.zip\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "99ad3a5bd7bb4fb79995029e23d2176e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /work/esm-vfc-catalogs/esm_vfc_data/FESOM2_PI_MESH/pi.zip\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "daf2ef2784ed4e44a119337d2ccf912e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"for entry in [cat[name] for name in cat]:\n",
" download_zenodo_files_for_entry(\n",
" entry,\n",
" force_download=True\n",
" )"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Read catalog entries"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<xarray.Dataset>\n",
"Dimensions: (elem: 5839, nod2: 3140, nz: 48, nz1: 47, time: 12)\n",
"Coordinates:\n",
" * time (time) float64 2.678e+06 5.097e+06 ... 2.886e+07 3.154e+07\n",
"Dimensions without coordinates: elem, nod2, nz, nz1\n",
"Data variables:\n",
" Av (time, elem, nz) float32 0.005085066 0.099402376 ... 0.0 0.0\n",
" Kv (time, nod2, nz) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n",
" MLD1 (time, nod2) float32 -75.19909 -77.151375 ... -33.665993 -32.839382\n",
" a_ice (time, nod2) float32 0.98289865 0.9805132 0.9803011 ... 0.0 0.0 0.0\n",
" m_ice (time, nod2) float32 1.1300349 1.1106514 1.1073328 ... 0.0 0.0 0.0\n",
" salt (time, nod2, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n",
" ssh (time, nod2) float32 -0.5813815 -0.5792128 ... -1.5763197 -1.586672\n",
" sst (time, nod2) float32 nan nan nan ... -1.517087 -1.5979521 -1.535592\n",
" temp (time, nod2, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n",
" u (time, elem, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n",
" uice (time, nod2) float32 -0.0049194503 -0.0022439826 ... 0.0 0.0\n",
" unod (time, nod2, nz1) float64 0.0008557 0.001066 0.001318 ... 0.0 0.0\n",
" v (time, elem, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n",
" vice (time, nod2) float32 -0.0016801578 -0.0011981231 ... 0.0 0.0\n",
" vnod (time, nod2, nz1) float64 0.0006255 0.0005963 0.0005808 ... 0.0 0.0\n",
" w (time, nod2, nz) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n"
]
}
],
"source": [
"print(cat[\"FESOM2_sample\"].read())"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>topo</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>-672</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>-534</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>-621</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>-744</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>-594</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3135</th>\n",
" <td>-1200</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3136</th>\n",
" <td>-850</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3137</th>\n",
" <td>-560</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3138</th>\n",
" <td>-121</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3139</th>\n",
" <td>-209</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3140 rows × 1 columns</p>\n",
"</div>"
],
"text/plain": [
" topo\n",
"0 -672\n",
"1 -534\n",
"2 -621\n",
"3 -744\n",
"4 -594\n",
"... ...\n",
"3135 -1200\n",
"3136 -850\n",
"3137 -560\n",
"3138 -121\n",
"3139 -209\n",
"\n",
"[3140 rows x 1 columns]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat[\"MESH_AUX3D\"].read()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" node_number x y flag\n",
"0 1 267.4665 84.5252 0\n",
"1 2 270.9702 84.3700 0\n",
"2 3 268.3207 84.0503 0\n",
"3 4 263.8669 84.5900 0\n",
"4 5 264.3323 84.2140 0\n",
"... ... ... ... ...\n",
"3135 3136 139.7089 -79.0173 0\n",
"3136 3137 144.1223 -79.2533 0\n",
"3137 3138 148.0000 -80.3240 1\n",
"3138 3139 135.7037 -79.9329 1\n",
"3139 3140 141.5839 -80.1193 1\n",
"\n",
"[3140 rows x 4 columns]\n"
]
}
],
"source": [
"print(cat[\"MESH_NOD2D\"].read())"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" first_elem second_elem third_elem\n",
"0 1 12 2\n",
"1 2 12 10\n",
"2 2 10 9\n",
"3 3 1 2\n",
"4 3 5 1\n",
"... ... ... ...\n",
"5834 3138 3137 3132\n",
"5835 3138 3131 3133\n",
"5836 3139 3136 3140\n",
"5837 3139 3135 3134\n",
"5838 3140 3137 3138\n",
"\n",
"[5839 rows x 3 columns]\n"
]
}
],
"source": [
"print(cat[\"MESH_ELEM2D\"].read())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}