{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Example using dask-gateway with a JupyterHub sandbox, intake-esm (on S3 NetCDF)\n", "\n", "\n", "Author: A.Radhakrishnan, Feb 2021" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "using t2.xlarge" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [], "source": [ "from netCDF4 import Dataset\n", "import numpy as np\n", "import matplotlib\n", "from matplotlib import pyplot as plt\n", "import xarray as xr\n", "import intake,yaml\n", "import intake_esm\n", "import numpy as np\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "#%pip install ipywidgets" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import sys\n", "from dask_gateway import Gateway\n", "gateway = Gateway()\n", "clusters = gateway.list_clusters()\n", "clusters" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [], "source": [ "if len(clusters) >0:\n", " cluster = gateway.connect(clusters[0].name)\n", " print(\"using existing connection\")\n", "else:\n", " from dask_gateway import GatewayCluster\n", " cluster = GatewayCluster()" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n",
"Client\n", "
| \n",
"\n",
"Cluster\n", "
| \n",
"
\n", "\n", "
\n", " | project | \n", "institute | \n", "model | \n", "experiment_id | \n", "frequency | \n", "modeling_realm | \n", "mip_table | \n", "ensemble_member | \n", "grid_label | \n", "variable | \n", "temporal subset | \n", "version | \n", "path | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "CMIP6 | \n", "AS-RCEC | \n", "TaiESM1 | \n", "histSST-piNTCF | \n", "NaN | \n", "NaN | \n", "AERmon | \n", "r1i1p1f1 | \n", "gn | \n", "ps | \n", "185001-201412 | \n", "v20200318 | \n", "s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... | \n", "
1 | \n", "CMIP6 | \n", "AS-RCEC | \n", "TaiESM1 | \n", "histSST-piNTCF | \n", "NaN | \n", "NaN | \n", "CFmon | \n", "r1i1p1f1 | \n", "gn | \n", "ta | \n", "185001-201412 | \n", "v20200318 | \n", "s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... | \n", "
2 | \n", "CMIP6 | \n", "AS-RCEC | \n", "TaiESM1 | \n", "histSST-piNTCF | \n", "NaN | \n", "NaN | \n", "Lmon | \n", "r1i1p1f1 | \n", "gn | \n", "rh | \n", "185002-201412 | \n", "v20200318 | \n", "s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... | \n", "
3 | \n", "CMIP6 | \n", "AS-RCEC | \n", "TaiESM1 | \n", "histSST | \n", "NaN | \n", "NaN | \n", "AERmon | \n", "r1i1p1f1 | \n", "gn | \n", "ps | \n", "185001-201412 | \n", "v20200310 | \n", "s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... | \n", "
4 | \n", "CMIP6 | \n", "AS-RCEC | \n", "TaiESM1 | \n", "histSST | \n", "NaN | \n", "NaN | \n", "CFmon | \n", "r1i1p1f1 | \n", "gn | \n", "ta | \n", "185001-201412 | \n", "v20200316 | \n", "s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
424087 | \n", "CMIP6 | \n", "THU | \n", "CIESM | \n", "ssp585 | \n", "mon | \n", "atmos | \n", "Amon | \n", "r1i1p1f1 | \n", "gr | \n", "rlds | \n", "201501-210012 | \n", "v20200417 | \n", "s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... | \n", "
424088 | \n", "CMIP6 | \n", "THU | \n", "CIESM | \n", "ssp585 | \n", "mon | \n", "atmos | \n", "Amon | \n", "r1i1p1f1 | \n", "gr | \n", "rsds | \n", "201501-210012 | \n", "v20200417 | \n", "s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... | \n", "
424089 | \n", "CMIP6 | \n", "THU | \n", "CIESM | \n", "ssp585 | \n", "mon | \n", "atmos | \n", "Amon | \n", "r1i1p1f1 | \n", "gr | \n", "ta | \n", "201501-210012 | \n", "v20200417 | \n", "s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... | \n", "
424090 | \n", "CMIP6 | \n", "THU | \n", "CIESM | \n", "ssp585 | \n", "mon | \n", "ocean | \n", "Omon | \n", "r1i1p1f1 | \n", "gn | \n", "thetao | \n", "201501-206412 | \n", "v20200220 | \n", "s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... | \n", "
424091 | \n", "CMIP6 | \n", "THU | \n", "CIESM | \n", "ssp585 | \n", "mon | \n", "ocean | \n", "Omon | \n", "r1i1p1f1 | \n", "gn | \n", "thetao | \n", "206501-210012 | \n", "v20200220 | \n", "s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... | \n", "
424092 rows × 13 columns
\n", "" ], "text/plain": [ " project institute model experiment_id frequency modeling_realm \\\n", "0 CMIP6 AS-RCEC TaiESM1 histSST-piNTCF NaN NaN \n", "1 CMIP6 AS-RCEC TaiESM1 histSST-piNTCF NaN NaN \n", "2 CMIP6 AS-RCEC TaiESM1 histSST-piNTCF NaN NaN \n", "3 CMIP6 AS-RCEC TaiESM1 histSST NaN NaN \n", "4 CMIP6 AS-RCEC TaiESM1 histSST NaN NaN \n", "... ... ... ... ... ... ... \n", "424087 CMIP6 THU CIESM ssp585 mon atmos \n", "424088 CMIP6 THU CIESM ssp585 mon atmos \n", "424089 CMIP6 THU CIESM ssp585 mon atmos \n", "424090 CMIP6 THU CIESM ssp585 mon ocean \n", "424091 CMIP6 THU CIESM ssp585 mon ocean \n", "\n", " mip_table ensemble_member grid_label variable temporal subset \\\n", "0 AERmon r1i1p1f1 gn ps 185001-201412 \n", "1 CFmon r1i1p1f1 gn ta 185001-201412 \n", "2 Lmon r1i1p1f1 gn rh 185002-201412 \n", "3 AERmon r1i1p1f1 gn ps 185001-201412 \n", "4 CFmon r1i1p1f1 gn ta 185001-201412 \n", "... ... ... ... ... ... \n", "424087 Amon r1i1p1f1 gr rlds 201501-210012 \n", "424088 Amon r1i1p1f1 gr rsds 201501-210012 \n", "424089 Amon r1i1p1f1 gr ta 201501-210012 \n", "424090 Omon r1i1p1f1 gn thetao 201501-206412 \n", "424091 Omon r1i1p1f1 gn thetao 206501-210012 \n", "\n", " version path \n", "0 v20200318 s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... \n", "1 v20200318 s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... \n", "2 v20200318 s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... \n", "3 v20200310 s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... \n", "4 v20200316 s3://esgf-world/CMIP6/AerChemMIP/AS-RCEC/TaiES... \n", "... ... ... \n", "424087 v20200417 s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... \n", "424088 v20200417 s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... \n", "424089 v20200417 s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... \n", "424090 v20200220 s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... \n", "424091 v20200220 s3://esgf-world/CMIP6/ScenarioMIP/THU/CIESM/ss... \n", "\n", "[424092 rows x 13 columns]" ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "col.df" ] }, { "cell_type": "code", "execution_count": 74, "metadata": {}, "outputs": [], "source": [ "#Examples to just search for what we want from the catalog\n", "expname_filter = ['historical']\n", "table_id_filter = 'Amon'\n", "model_filter = 'GFDL-ESM4'\n", "variable_id_filter = \"tas\"\n", "ens_filter = \"r1i1p1f1\"\n", "version_filter = \"v20190726\"\n", "cat = col.search(experiment_id=expname_filter, mip_table=table_id_filter,model=model_filter,variable=variable_id_filter,version=\"v20190726\")" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "--> The keys in the returned dictionary of datasets are constructed as follows:\n", "\t'project.institute.model.experiment_id.mip_table'\n" ] }, { "data": { "text/html": [ "\n", "\n", " | project | \n", "institute | \n", "model | \n", "experiment_id | \n", "frequency | \n", "modeling_realm | \n", "mip_table | \n", "ensemble_member | \n", "grid_label | \n", "variable | \n", "temporal subset | \n", "version | \n", "path | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "CMIP6 | \n", "NOAA-GFDL | \n", "GFDL-ESM4 | \n", "historical | \n", "mon | \n", "atmos | \n", "Amon | \n", "r1i1p1f1 | \n", "gr1 | \n", "tas | \n", "185001-194912 | \n", "v20190726 | \n", "s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4... | \n", "
1 | \n", "CMIP6 | \n", "NOAA-GFDL | \n", "GFDL-ESM4 | \n", "historical | \n", "mon | \n", "atmos | \n", "Amon | \n", "r1i1p1f1 | \n", "gr1 | \n", "tas | \n", "195001-201412 | \n", "v20190726 | \n", "s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4... | \n", "