{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Intake Zenodo Fetcher\n\nHere, we'll use `intake_zenodo_fetcher` to populate the location an intake catalog points to with data from Zenodo.\nFor this to work, we need the catalog entries to contain a `metadata.zenod_doi` key.\nThis can increase reproducibility of workflows that use data archived on Zenodo.\n\nAn example for this FESOM2 example grid (<https://zenodo.org/record/3865567>) is:\n```yaml\nmetadata:\n  version: 1\n\nplugins:\n  source:\n      - module: intake_xarray\n\nsources:\n  FESOM2_sample:\n    driver: netcdf\n    description: 'FESOM2 pi mesh Sample dataset'\n    metadata:\n      zenodo_doi: \"10.5281/zenodo.3865567\"\n    args:\n      urlpath: \"{{ CATALOG_DIR }}/FESOM2_PI_MESH/*.fesom.1948.nc\"\n      xarray_kwargs:\n        decode_cf: False\n        combine: 'by_coords'\n\n```\nThen,\n```python\nintake_zenodo_fetcher.download_zenodo_files_for_entry(\n    cat['FESOM2_sample']\n)\n```\nwill make sure all files matching the glob pattern `\"*.fesom.1948.nc\"` are downloaded to `./FESOM2_PI_MESH/` in the directory that also holds the catalog."},{"metadata":{},"cell_type":"markdown","source":"## Technical Preamble"},{"metadata":{"trusted":true},"cell_type":"code","source":"import intake\nfrom pathlib import Path\nfrom intake_zenodo_fetcher import download_zenodo_files_for_entry","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# parameters\ncatalog_file = Path(\"catalogs/FESOM2_PI_MESH.yaml\")","execution_count":2,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Open Catalog"},{"metadata":{"trusted":true},"cell_type":"code","source":"catalog = intake.open_catalog(str(catalog_file))","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list(catalog)","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"['FESOM2_sample', 'MESH_NOD2D', 'MESH_ELEM2D', 'MESH_AUX3D']"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(catalog[\"FESOM2_sample\"])","execution_count":5,"outputs":[{"output_type":"display_data","data":{"application/json":"FESOM2_sample:\n  args:\n    urlpath: /home/jovyan/examples/catalogs//FESOM2_PI_MESH/*.fesom.1948.nc\n    xarray_kwargs:\n      combine: by_coords\n      decode_cf: false\n  description: FESOM2 pi mesh Sample dataset\n  driver: intake_xarray.netcdf.NetCDFSource\n  metadata:\n    catalog_dir: /home/jovyan/examples/catalogs/\n    zenodo_doi: 10.5281/zenodo.3865567\n","text/plain":"FESOM2_sample:\n  args:\n    urlpath: /home/jovyan/examples/catalogs//FESOM2_PI_MESH/*.fesom.1948.nc\n    xarray_kwargs:\n      combine: by_coords\n      decode_cf: false\n  description: FESOM2 pi mesh Sample dataset\n  driver: intake_xarray.netcdf.NetCDFSource\n  metadata:\n    catalog_dir: /home/jovyan/examples/catalogs/\n    zenodo_doi: 10.5281/zenodo.3865567\n"},"metadata":{"application/json":{"root":"FESOM2_sample"}}}]},{"metadata":{},"cell_type":"markdown","source":"## Get Local Copy of the Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for catalog_entry in list(catalog):\n    download_zenodo_files_for_entry(\n        catalog[catalog_entry],\n        force_download=True\n    )","execution_count":6,"outputs":[{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/a_ice.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/a_ice.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"557f47f8b0f74ea5a63b01edeb1c7edd"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/Av.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/Av.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc638b6b9b2141b58a8cf173ffbed016"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/Kv.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/Kv.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"258096d131c043a9857ff61a88887ade"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/m_ice.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/m_ice.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f22d6fad056416891034b6d5ec85eae"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/MLD1.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/MLD1.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b95f68e0a86e492ab8d82a07db548413"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/salt.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/salt.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4b4a6d23792f4ae5ab5f84b988931303"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/ssh.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/ssh.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4df1e1d39e8045df8deef07dc1387765"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/sst.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/sst.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"76ee9cfddbbb4b4da22b2520faf0fa9c"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/temp.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/temp.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"49641084f67b4f97ac3e9c8de06a1af8"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/u.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/u.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a6d1312ba2d24081bb440ec39e21c860"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/uice.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/uice.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b14ba86e88c24e29a7156a018d0444d8"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/unod.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/unod.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0c316ac04a9f4a46afcd5864b22a7756"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/v.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/v.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"21766c0d08a24186ab27c4fb17754d33"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/vice.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/vice.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2b73ae60bdf54d93b7cae1218190b1d8"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/vnod.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/vnod.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"917d2b9414224e388707cefdc3150f38"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/w.fesom.1948.nc to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/w.fesom.1948.nc\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1377bc6a4120421a962c7a955d796067"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/pi.zip\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bff0f272ba6446b6aebc6f46a203c75b"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/pi.zip\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9fada63708de42a186c48525c68eb75b"}},"metadata":{}},{"output_type":"stream","text":"will download https://zenodo.org/api/files/21cd530d-24d2-46bb-94c5-dc0ba924b5a6/pi.zip to /home/jovyan/examples/catalogs/FESOM2_PI_MESH/pi.zip\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0.00/100 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6238ef25d050428e8f3ac5ac00fe2f5d"}},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"## Use Local Copy of the Data\n\nNow we have fetched a local copy of all files from the Zenodo record <https://zenodo.org/record/3865567> that are relevant for our catalog.\n\nLet's make sure they are actually usable by reading them."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(catalog[\"FESOM2_sample\"].read())","execution_count":7,"outputs":[{"output_type":"stream","text":"<xarray.Dataset>\nDimensions:  (elem: 5839, nod2: 3140, nz: 48, nz1: 47, time: 12)\nCoordinates:\n  * time     (time) float64 2.678e+06 5.097e+06 ... 2.886e+07 3.154e+07\nDimensions without coordinates: elem, nod2, nz, nz1\nData variables:\n    Av       (time, elem, nz) float32 0.005085 0.0994 0.09765 ... 0.0 0.0 0.0\n    Kv       (time, nod2, nz) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n    MLD1     (time, nod2) float32 -75.2 -77.15 -80.05 ... -32.84 -33.67 -32.84\n    a_ice    (time, nod2) float32 0.9829 0.9805 0.9803 0.9887 ... 0.0 0.0 0.0\n    m_ice    (time, nod2) float32 1.13 1.111 1.107 1.163 ... 0.0 0.0 0.0 0.0\n    salt     (time, nod2, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n    ssh      (time, nod2) float32 -0.5814 -0.5792 -0.5769 ... -1.576 -1.587\n    sst      (time, nod2) float32 nan nan nan nan ... -1.517 -1.598 -1.536\n    temp     (time, nod2, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n    u        (time, elem, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n    uice     (time, nod2) float32 -0.004919 -0.002244 0.0007494 ... 0.0 0.0 0.0\n    unod     (time, nod2, nz1) float64 0.0008557 0.001066 0.001318 ... 0.0 0.0\n    v        (time, elem, nz1) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n    vice     (time, nod2) float32 -0.00168 -0.001198 0.0005354 ... 0.0 0.0 0.0\n    vnod     (time, nod2, nz1) float64 0.0006255 0.0005963 0.0005808 ... 0.0 0.0\n    w        (time, nod2, nz) float32 nan nan nan nan nan ... 0.0 0.0 0.0 0.0\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(catalog[\"MESH_AUX3D\"].read())","execution_count":8,"outputs":[{"output_type":"stream","text":"      topo\n0     -672\n1     -534\n2     -621\n3     -744\n4     -594\n...    ...\n3135 -1200\n3136  -850\n3137  -560\n3138  -121\n3139  -209\n\n[3140 rows x 1 columns]\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(catalog[\"MESH_NOD2D\"].read())","execution_count":9,"outputs":[{"output_type":"stream","text":"      node_number         x        y  flag\n0               1  267.4665  84.5252     0\n1               2  270.9702  84.3700     0\n2               3  268.3207  84.0503     0\n3               4  263.8669  84.5900     0\n4               5  264.3323  84.2140     0\n...           ...       ...      ...   ...\n3135         3136  139.7089 -79.0173     0\n3136         3137  144.1223 -79.2533     0\n3137         3138  148.0000 -80.3240     1\n3138         3139  135.7037 -79.9329     1\n3139         3140  141.5839 -80.1193     1\n\n[3140 rows x 4 columns]\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(catalog[\"MESH_ELEM2D\"].read())","execution_count":10,"outputs":[{"output_type":"stream","text":"      first_elem  second_elem  third_elem\n0              1           12           2\n1              2           12          10\n2              2           10           9\n3              3            1           2\n4              3            5           1\n...          ...          ...         ...\n5834        3138         3137        3132\n5835        3138         3131        3133\n5836        3139         3136        3140\n5837        3139         3135        3134\n5838        3140         3137        3138\n\n[5839 rows x 3 columns]\n","name":"stdout"}]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}