{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# How to use GCRCatalogs\n", "\n", "by Yao-Yuan Mao (last update: 7/27/2018)\n", "\n", "Links to GitHub code repos: [GCRCatalogs](https://github.com/LSSTDESC/gcr-catalogs) and [GCR](https://github.com/yymao/generic-catalog-reader)\n", "\n", "**Note: You should be running this notebook at https://jupyter-dev.nersc.gov **" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "## Note: if you clone the gcr-catalogs repo and are running this under the `examples` folder,\n", "## you can also use your version of GCRCatalogs:\n", "#import sys\n", "#sys.path.insert(0, '/path/to/your/cloned/gcr-catalogs')\n", "\n", "## The following lines are to check if you're in the lsst group\n", "import subprocess\n", "assert u'lsst' in subprocess.check_output(['groups']).decode().split(), 'You need to be in the `lsst` group for this notebook to work'" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import numpy as np" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## The basics:\n", "\n", "- `get_available_catalogs()` lists available catlaogs; returns `dict`.\n", "- `load_catalog()` loads the catalog you want; returns an instance of `GCR.BaseGenericCatalog`" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GCRCatalogs = 0.8.0 | GCR = 0.7.2\n" ] } ], "source": [ "import GCRCatalogs\n", "\n", "## check version\n", "print('GCRCatalogs =', GCRCatalogs.__version__, '|' ,'GCR =', GCRCatalogs.GCR.__version__)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "buzzard\n", "buzzard_high-res\n", "buzzard_test\n", "dc1\n", "dc2_coadd_run1.1p\n", "dc2_coadd_run1.1p_tract4850\n", "focal_plane_0_test\n", "focal_plane_16_test\n", "hsc-pdr1-xmm\n", "protoDC2\n" ] } ], "source": [ "## find available catlaogs, sorted by their name\n", "\n", "print('\\n'.join(sorted(GCRCatalogs.get_available_catalogs())))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "buzzard\n", "buzzard_high-res\n", "buzzard_high-res_v1.1\n", "buzzard_test\n", "buzzard_v1.6\n", "buzzard_v1.6_1\n", "buzzard_v1.6_2\n", "buzzard_v1.6_21\n", "buzzard_v1.6_3\n", "buzzard_v1.6_5\n", "buzzard_v1.6_test\n", "cosmoDC2_v0.1\n", "cosmoDC2_v0.1_test\n", "dc1\n", "dc2_coadd_run1.1p\n", "dc2_coadd_run1.1p_tract4850\n", "dc2_instance_example1\n", "dc2_instance_example2\n", "dc2_reference_run1.1\n", "dc2_reference_run1.2\n", "dc2_truth_run1.1\n", "focal_plane_0_test\n", "focal_plane_16_test\n", "hsc-pdr1-xmm\n", "proto-dc2_v2.0\n", "proto-dc2_v2.0_redmapper\n", "proto-dc2_v2.0_test\n", "proto-dc2_v2.1\n", "proto-dc2_v2.1.1\n", "proto-dc2_v2.1.2\n", "proto-dc2_v2.1.2_addon_knots\n", "proto-dc2_v2.1.2_test\n", "proto-dc2_v3.0\n", "proto-dc2_v3.0_addon_knots\n", "proto-dc2_v3.0_redmapper\n", "proto-dc2_v3.0_test\n", "proto-dc2_v4.3_redmapper\n", "proto-dc2_v4.4\n", "proto-dc2_v4.4_test\n", "proto-dc2_v4.5\n", "proto-dc2_v4.5_test\n", "proto-dc2_v4.6.1\n", "proto-dc2_v4.6.1_test\n", "proto-dc2_v4.7_test\n", "proto-dc2_v5.0\n", "proto-dc2_v5.0_test\n", "protoDC2\n", "protoDC2_addon_tidal\n", "protoDC2_test\n", "um_v0.1\n", "um_v0.1_shear_test\n", "um_v0.1_test\n" ] } ], "source": [ "## find *all* available catlaogs, including some older versions, sorted by their name\n", "\n", "print('\\n'.join(sorted(GCRCatalogs.get_available_catalogs(include_default_only=False))))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "proto-dc2_v2.0\n", "proto-dc2_v2.0_redmapper\n", "proto-dc2_v2.0_test\n", "proto-dc2_v2.1\n", "proto-dc2_v2.1.1\n", "proto-dc2_v2.1.2\n", "proto-dc2_v2.1.2_addon_knots\n", "proto-dc2_v2.1.2_test\n", "proto-dc2_v3.0\n", "proto-dc2_v3.0_addon_knots\n", "proto-dc2_v3.0_redmapper\n", "proto-dc2_v3.0_test\n", "proto-dc2_v4.3_redmapper\n", "proto-dc2_v4.4\n", "proto-dc2_v4.4_test\n", "proto-dc2_v4.5\n", "proto-dc2_v4.5_test\n", "proto-dc2_v4.6.1\n", "proto-dc2_v4.6.1_test\n", "proto-dc2_v4.7_test\n", "proto-dc2_v5.0\n", "proto-dc2_v5.0_test\n" ] } ], "source": [ "## find *all* available catlaogs whose name starts with 'proto-dc2', sorted by their name\n", "\n", "print('\\n'.join(sorted(c for c in GCRCatalogs.get_available_catalogs(include_default_only=False) if c.startswith('proto-dc2'))))" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/global/common/software/lsst/common/miniconda/current/lib/python3.6/site-packages/GCRCatalogs/alphaq.py:105: UserWarning: No md5 sum specified in the config file\n", " warnings.warn('No md5 sum specified in the config file')\n" ] } ], "source": [ "## load 'protoDC2' catalog\n", "\n", "gc = GCRCatalogs.load_catalog('protoDC2_test') # use 'protoDC2_test' to skip md5 check (which takes a while)\n", "#gc = GCRCatalogs.load_catalog('protoDC2')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## GCR interface\n", "\n", "See also the [full GCR API Documentation](https://yymao.github.io/generic-catalog-reader/index.html)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### quantities\n", "\n", "- `get_quantities()` loads the quantities you need; takes a `list` and returns `dict`.\n", "- `has_quantity()` and `has_quantities()` can check if the quantities you need exist; both return `bool`.\n", "- `list_all_quantities()` lists all available catlaogs; returns `list`." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'dec': array([-1.78091323, -2.06204367, -1.48433971, ..., 2.43747878,\n", " 2.49400425, 2.37360001], dtype=float32),\n", " 'ra': array([-0.21326847, -1.29391468, 0.68152297, ..., -2.39075041,\n", " -2.46616554, -2.31633687], dtype=float32),\n", " 'mag_u_lsst': array([ 21.2349205 , 21.83907318, 21.02371979, ..., 26.76194382,\n", " 33.03851318, 28.94227791], dtype=float32)}" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.get_quantities(['mag_u_lsst', 'ra', 'dec'])" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.has_quantity('mag_u_lsst')" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.has_quantities(['mag_u_lsst', 'ra', 'dec'])" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "False" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "## Only returns `True` if *all* quantities exist\n", "gc.has_quantities(['mag_u_lsst', 'ra', 'dec', 'quantitiy_that_does_not_exist'])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "A_v, A_v_bulge, A_v_disk, Mag_true_Y_lsst_z0, Mag_true_Y_lsst_z0_no_host_extinction, Mag_true_g_lsst_z0, Mag_true_g_lsst_z0_no_host_extinction, Mag_true_g_sdss_z0, Mag_true_g_sdss_z0_no_host_extinction, Mag_true_i_lsst_z0, Mag_true_i_lsst_z0_no_host_extinction, Mag_true_i_sdss_z0, Mag_true_i_sdss_z0_no_host_extinction, Mag_true_r_lsst_z0, Mag_true_r_lsst_z0_no_host_extinction, Mag_true_r_sdss_z0, Mag_true_r_sdss_z0_no_host_extinction, Mag_true_u_lsst_z0, Mag_true_u_lsst_z0_no_host_extinction, Mag_true_u_sdss_z0, Mag_true_u_sdss_z0_no_host_extinction, Mag_true_y_lsst_z0, Mag_true_y_lsst_z0_no_host_extinction, Mag_true_z_lsst_z0, Mag_true_z_lsst_z0_no_host_extinction, Mag_true_z_sdss_z0, Mag_true_z_sdss_z0_no_host_extinction, R_v, R_v_bulge, R_v_disk, bulge_to_total_ratio_i, convergence, dec, dec_true, ellipticity_1_bulge_true, ellipticity_1_disk_true, ellipticity_1_true, ellipticity_2_bulge_true, ellipticity_2_disk_true, ellipticity_2_true, ellipticity_bulge_true, ellipticity_disk_true, ellipticity_true, galaxy_id, halo_id, halo_mass, is_central, mag_Y_lsst, mag_Y_lsst_no_host_extinction, mag_g, mag_g_lsst, mag_g_lsst_no_host_extinction, mag_g_sdss, mag_g_sdss_no_host_extinction, mag_i, mag_i_lsst, mag_i_lsst_no_host_extinction, mag_i_sdss, mag_i_sdss_no_host_extinction, mag_r, mag_r_lsst, mag_r_lsst_no_host_extinction, mag_r_sdss, mag_r_sdss_no_host_extinction, mag_true_Y_lsst, mag_true_Y_lsst_no_host_extinction, mag_true_g, mag_true_g_lsst, mag_true_g_lsst_no_host_extinction, mag_true_g_sdss, mag_true_g_sdss_no_host_extinction, mag_true_i, mag_true_i_lsst, mag_true_i_lsst_no_host_extinction, mag_true_i_sdss, mag_true_i_sdss_no_host_extinction, mag_true_r, mag_true_r_lsst, mag_true_r_lsst_no_host_extinction, mag_true_r_sdss, mag_true_r_sdss_no_host_extinction, mag_true_u, mag_true_u_lsst, mag_true_u_lsst_no_host_extinction, mag_true_u_sdss, mag_true_u_sdss_no_host_extinction, mag_true_y, mag_true_y_lsst, mag_true_y_lsst_no_host_extinction, mag_true_z, mag_true_z_lsst, mag_true_z_lsst_no_host_extinction, mag_true_z_sdss, mag_true_z_sdss_no_host_extinction, mag_u, mag_u_lsst, mag_u_lsst_no_host_extinction, mag_u_sdss, mag_u_sdss_no_host_extinction, mag_y, mag_y_lsst, mag_y_lsst_no_host_extinction, mag_z, mag_z_lsst, mag_z_lsst_no_host_extinction, mag_z_sdss, mag_z_sdss_no_host_extinction, magnification, position_angle_true, position_x, position_y, position_z, ra, ra_true, redshift, redshift_true, sed_1000_246, sed_1000_246_bulge, sed_1000_246_bulge_no_host_extinction, sed_1000_246_disk, sed_1000_246_disk_no_host_extinction, sed_1000_246_no_host_extinction, sed_11467_1710, sed_11467_1710_bulge, sed_11467_1710_bulge_no_host_extinction, sed_11467_1710_disk, sed_11467_1710_disk_no_host_extinction, sed_11467_1710_no_host_extinction, sed_1246_306, sed_1246_306_bulge, sed_1246_306_bulge_no_host_extinction, sed_1246_306_disk, sed_1246_306_disk_no_host_extinction, sed_1246_306_no_host_extinction, sed_13177_1966, sed_13177_1966_bulge, sed_13177_1966_bulge_no_host_extinction, sed_13177_1966_disk, sed_13177_1966_disk_no_host_extinction, sed_13177_1966_no_host_extinction, sed_15143_2259, sed_15143_2259_bulge, sed_15143_2259_bulge_no_host_extinction, sed_15143_2259_disk, sed_15143_2259_disk_no_host_extinction, sed_15143_2259_no_host_extinction, sed_1552_381, sed_1552_381_bulge, sed_1552_381_bulge_no_host_extinction, sed_1552_381_disk, sed_1552_381_disk_no_host_extinction, sed_1552_381_no_host_extinction, sed_17402_2596, sed_17402_2596_bulge, sed_17402_2596_bulge_no_host_extinction, sed_17402_2596_disk, sed_17402_2596_disk_no_host_extinction, sed_17402_2596_no_host_extinction, sed_1933_474, sed_1933_474_bulge, sed_1933_474_bulge_no_host_extinction, sed_1933_474_disk, sed_1933_474_disk_no_host_extinction, sed_1933_474_no_host_extinction, sed_2407_591, sed_2407_591_bulge, sed_2407_591_bulge_no_host_extinction, sed_2407_591_disk, sed_2407_591_disk_no_host_extinction, sed_2407_591_no_host_extinction, sed_2998_186, sed_2998_186_bulge, sed_2998_186_bulge_no_host_extinction, sed_2998_186_disk, sed_2998_186_disk_no_host_extinction, sed_2998_186_no_host_extinction, sed_3184_197, sed_3184_197_bulge, sed_3184_197_bulge_no_host_extinction, sed_3184_197_disk, sed_3184_197_disk_no_host_extinction, sed_3184_197_no_host_extinction, sed_3381_209, sed_3381_209_bulge, sed_3381_209_bulge_no_host_extinction, sed_3381_209_disk, sed_3381_209_disk_no_host_extinction, sed_3381_209_no_host_extinction, sed_3590_222, sed_3590_222_bulge, sed_3590_222_bulge_no_host_extinction, sed_3590_222_disk, sed_3590_222_disk_no_host_extinction, sed_3590_222_no_host_extinction, sed_3812_236, sed_3812_236_bulge, sed_3812_236_bulge_no_host_extinction, sed_3812_236_disk, sed_3812_236_disk_no_host_extinction, sed_3812_236_no_host_extinction, sed_4048_251, sed_4048_251_bulge, sed_4048_251_bulge_no_host_extinction, sed_4048_251_disk, sed_4048_251_disk_no_host_extinction, sed_4048_251_no_host_extinction, sed_4299_266, sed_4299_266_bulge, sed_4299_266_bulge_no_host_extinction, sed_4299_266_disk, sed_4299_266_disk_no_host_extinction, sed_4299_266_no_host_extinction, sed_4565_283, sed_4565_283_bulge, sed_4565_283_bulge_no_host_extinction, sed_4565_283_disk, sed_4565_283_disk_no_host_extinction, sed_4565_283_no_host_extinction, sed_4848_300, sed_4848_300_bulge, sed_4848_300_bulge_no_host_extinction, sed_4848_300_disk, sed_4848_300_disk_no_host_extinction, sed_4848_300_no_host_extinction, sed_5148_319, sed_5148_319_bulge, sed_5148_319_bulge_no_host_extinction, sed_5148_319_disk, sed_5148_319_disk_no_host_extinction, sed_5148_319_no_host_extinction, sed_5467_339, sed_5467_339_bulge, sed_5467_339_bulge_no_host_extinction, sed_5467_339_disk, sed_5467_339_disk_no_host_extinction, sed_5467_339_no_host_extinction, sed_5806_360, sed_5806_360_bulge, sed_5806_360_bulge_no_host_extinction, sed_5806_360_disk, sed_5806_360_disk_no_host_extinction, sed_5806_360_no_host_extinction, sed_6166_382, sed_6166_382_bulge, sed_6166_382_bulge_no_host_extinction, sed_6166_382_disk, sed_6166_382_disk_no_host_extinction, sed_6166_382_no_host_extinction, sed_6548_406, sed_6548_406_bulge, sed_6548_406_bulge_no_host_extinction, sed_6548_406_disk, sed_6548_406_disk_no_host_extinction, sed_6548_406_no_host_extinction, sed_6954_431, sed_6954_431_bulge, sed_6954_431_bulge_no_host_extinction, sed_6954_431_disk, sed_6954_431_disk_no_host_extinction, sed_6954_431_no_host_extinction, sed_7385_458, sed_7385_458_bulge, sed_7385_458_bulge_no_host_extinction, sed_7385_458_disk, sed_7385_458_disk_no_host_extinction, sed_7385_458_no_host_extinction, sed_7843_486, sed_7843_486_bulge, sed_7843_486_bulge_no_host_extinction, sed_7843_486_disk, sed_7843_486_disk_no_host_extinction, sed_7843_486_no_host_extinction, sed_8329_517, sed_8329_517_bulge, sed_8329_517_bulge_no_host_extinction, sed_8329_517_disk, sed_8329_517_disk_no_host_extinction, sed_8329_517_no_host_extinction, sed_8846_549, sed_8846_549_bulge, sed_8846_549_bulge_no_host_extinction, sed_8846_549_disk, sed_8846_549_disk_no_host_extinction, sed_8846_549_no_host_extinction, sed_9395_583, sed_9395_583_bulge, sed_9395_583_bulge_no_host_extinction, sed_9395_583_disk, sed_9395_583_disk_no_host_extinction, sed_9395_583_no_host_extinction, sed_9978_1489, sed_9978_1489_bulge, sed_9978_1489_bulge_no_host_extinction, sed_9978_1489_disk, sed_9978_1489_disk_no_host_extinction, sed_9978_1489_no_host_extinction, sersic_bulge, sersic_disk, shear_1, shear_2, shear_2_phosim, shear_2_treecorr, size_bulge_true, size_disk_true, size_minor_bulge_true, size_minor_disk_true, size_minor_true, size_true, stellar_mass, stellar_mass_bulge, stellar_mass_disk, velocity_x, velocity_y, velocity_z\n" ] } ], "source": [ "print(', '.join(sorted(gc.list_all_quantities())))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### accessing native quantities\n", "\n", "Native quantities are quantities that have not yet be homogenized (to common labels/units).\n", "However, you can still access them as long as you know what you are doing. " ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "LSST_filters/diskLuminositiesStellar:LSST_g:observed\n", "LSST_filters/diskLuminositiesStellar:LSST_g:observed:dustAtlas\n", "LSST_filters/diskLuminositiesStellar:LSST_g:rest\n", "LSST_filters/diskLuminositiesStellar:LSST_g:rest:dustAtlas\n", "LSST_filters/diskLuminositiesStellar:LSST_i:observed\n" ] } ], "source": [ "## print out the first 5 native quantities\n", "\n", "print('\\n'.join(sorted(gc.list_all_native_quantities())[:5]))" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "A_v\n", "A_v_bulge\n", "A_v_disk\n", "LSST_filters/diskLuminositiesStellar:LSST_g:observed\n", "LSST_filters/diskLuminositiesStellar:LSST_g:observed:dustAtlas\n" ] } ], "source": [ "## list both native or derived quantities, print the first 5 out\n", "\n", "print('\\n'.join(sorted(gc.list_all_quantities(include_native=True))[:5]))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['sed_2998_186',\n", " 'sed_13177_1966',\n", " 'sed_15143_2259',\n", " 'sed_4048_251',\n", " 'sed_6548_406',\n", " 'sed_6166_382',\n", " 'sed_7385_458',\n", " 'sed_4565_283',\n", " 'sed_8846_549',\n", " 'sed_9978_1489',\n", " 'sed_5806_360',\n", " 'sed_7843_486',\n", " 'sed_6954_431',\n", " 'sed_11467_1710',\n", " 'sed_4848_300',\n", " 'sed_1552_381',\n", " 'sed_4299_266',\n", " 'sed_1000_246',\n", " 'sed_5148_319',\n", " 'sed_3184_197',\n", " 'sed_9395_583',\n", " 'sed_3381_209',\n", " 'sed_1246_306',\n", " 'sed_2407_591',\n", " 'sed_5467_339',\n", " 'sed_3812_236',\n", " 'sed_1933_474',\n", " 'sed_17402_2596',\n", " 'sed_8329_517',\n", " 'sed_3590_222']" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# find all quantities that match a regular expression\n", "import re\n", "\n", "data = gc.get_quantities([q for q in gc.list_all_quantities() if re.match(r'sed_\\d+_\\d+$', q)])\n", "list(data.keys())" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'LSST_filters/diskLuminositiesStellar:LSST_g:observed:dustAtlas': array([ 1035582.25 , 580861.5 , 980863.875 , ...,\n", " 3248385.5 , 38833.45703125, 442917.21875 ], dtype=float32),\n", " 'LSST_filters/diskLuminositiesStellar:LSST_g:observed': array([ 992498.5625 , 590357.75 , 1002641.3125 , ...,\n", " 3645767. , 40224.2265625, 485800.09375 ], dtype=float32)}" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# to retrive native quantities, you can just use `get_quantities` as usual\n", "gc.get_quantities(['LSST_filters/diskLuminositiesStellar:LSST_g:observed',\n", " 'LSST_filters/diskLuminositiesStellar:LSST_g:observed:dustAtlas'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "You can also rename the native quantities by using `add_quantity_modifier()`. For example:" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'balmer_alpha_6563': array([ 1.76363730e+04, 4.74945703e+03, 1.73469531e+05, ...,\n", " 7.58312000e+07, 4.01797803e+03, 6.08336523e+03], dtype=float32)}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# you can also make quantity alias\n", "\n", "gc.add_quantity_modifier('balmer_alpha_6563', 'emissionLines/diskLineLuminosity:balmerAlpha6563:rest')\n", "gc.get_quantities(['balmer_alpha_6563'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### more info about the catalog\n", "\n", "- `lightcone` is a `bool`\n", "- `cosmology` is a instance of `astropy.cosmology.FLRW`\n", "- `get_input_kwargs()` returns a `dict` (when no argument) from the orignal yaml config file. \n", " If a argument if passed, it returns the corresponding value for key=argument" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.lightcone" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "FlatLambdaCDM(H0=71 km / (Mpc s), Om0=0.265, Tcmb0=0 K, Neff=3.04, m_nu=None, Ob0=0.0448)" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.cosmology" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'5.0'" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gc.version" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ProtoDC2 is a down-scaled version of the catalog to be generated for LSST-DESC DC2.\n", "For a description of the catalog and the methods, please see https://goo.gl/fXDQwP\n", "\n" ] } ], "source": [ "print(gc.get_catalog_info('description'))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'subclass_name': 'alphaq.AlphaQGalaxyCatalog', 'filename': '/global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/v5.0.all.hdf5', 'lightcone': True, 'version': '5.0', 'creators': ['Andrew Benson', 'Andrew Hearin', 'Katrin Heitmann', 'Joe Hollowed', 'Danila Korytov', 'Eve Kovacs', 'Patricia Larsen'], 'description': 'ProtoDC2 is a down-scaled version of the catalog to be generated for LSST-DESC DC2.\\nFor a description of the catalog and the methods, please see https://goo.gl/fXDQwP\\n'}\n" ] } ], "source": [ "print(gc.get_catalog_info())" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "A_v None\n", "A_v_bulge None\n", "A_v_disk None\n", "Mag_true_Y_lsst_z0 {'units': 'AB magnitude'}\n", "Mag_true_Y_lsst_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_g_lsst_z0 {'units': 'AB magnitude'}\n", "Mag_true_g_lsst_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_g_sdss_z0 {'units': 'AB magnitude'}\n", "Mag_true_g_sdss_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_i_lsst_z0 {'units': 'AB magnitude'}\n", "Mag_true_i_lsst_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_i_sdss_z0 {'units': 'AB magnitude'}\n", "Mag_true_i_sdss_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_r_lsst_z0 {'units': 'AB magnitude'}\n", "Mag_true_r_lsst_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_r_sdss_z0 {'units': 'AB magnitude'}\n", "Mag_true_r_sdss_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_u_lsst_z0 {'units': 'AB magnitude'}\n", "Mag_true_u_lsst_z0_no_host_extinction {'units': 'AB magnitude'}\n", "Mag_true_u_sdss_z0 {'units': 'AB magnitude'}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/global/common/software/lsst/common/miniconda/current/lib/python3.6/site-packages/GCRCatalogs/alphaq.py:366: UserWarning: This value is composed of a function on native quantities. So we have no idea what the units are\n", " warnings.warn('This value is composed of a function on native quantities. So we have no idea what the units are')\n" ] } ], "source": [ "for q in sorted(gc.list_all_quantities())[:20]:\n", " print(q, gc.get_quantity_info(q))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### use filters\n", "\n", "You can specify `filters` in `get_quantities` to select a subset of data. \n", "Note that `filters` always takes a list." ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "True\n" ] } ], "source": [ "# note that we use a list even there is only one filter\n", "\n", "data = gc.get_quantities(['stellar_mass', 'ra', 'dec'], filters=['stellar_mass > 1e10']) \n", "print((data['stellar_mass'] > 1e10).all())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "570847\n", "570847\n" ] } ], "source": [ "## You can use more than one filter.\n", "\n", "data = gc.get_quantities(['stellar_mass'], filters=['ra < -2', 'dec > 1'])\n", "print(len(data['stellar_mass']))\n", "\n", "data_check = gc.get_quantities(['ra', 'dec'])\n", "print(np.count_nonzero((data_check['ra'] < -2) & (data_check['dec'] > 1)))" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "# For more complicated filters, specify them as tuple of (callable, quantity1, quantity2, ...)\n", "\n", "data = gc.get_quantities(['stellar_mass'], filters=[(np.isfinite, 'stellar_mass')])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### native filters\n", "\n", "Some catalogs (currently only buzzard and buzzard_high-res) support \"native filters\", which you can use to load only a subset of data more efficiently. " ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'healpix_pixel'}\n", "44.9959447517 56.2350032152 4.77918879056 14.4744615426\n", "33.7591549807 44.9914241079 4.77569650762 14.4781225874\n" ] } ], "source": [ "gc_buzzard = GCRCatalogs.load_catalog('buzzard')\n", "print(gc_buzzard._native_filter_quantities)\n", "\n", "data = gc_buzzard.get_quantities(['ra', 'dec'], native_filters=['healpix_pixel == 1'])\n", "print(data['ra'].min(), data['ra'].max(), data['dec'].min(), data['dec'].max())\n", "\n", "data = gc_buzzard.get_quantities(['ra', 'dec'], native_filters=['healpix_pixel == 2'])\n", "print(data['ra'].min(), data['ra'].max(), data['dec'].min(), data['dec'].max())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### more tips on using the quantities\n", "\n", "#### tip 1\n", "`get_quantities()` returns a dictionary, which you can easily turn into a `astropy.table.Table` or `pandas.DataFrame`" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [], "source": [ "data = gc.get_quantities(['mag_u_lsst', 'ra', 'dec'])" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", " | dec | \n", "mag_u_lsst | \n", "ra | \n", "
---|---|---|---|
0 | \n", "-1.780913 | \n", "21.234921 | \n", "-0.213268 | \n", "
1 | \n", "-2.062044 | \n", "21.839073 | \n", "-1.293915 | \n", "
2 | \n", "-1.484340 | \n", "21.023720 | \n", "0.681523 | \n", "
3 | \n", "-0.923260 | \n", "17.603844 | \n", "-0.304187 | \n", "
4 | \n", "-0.874825 | \n", "19.314825 | \n", "-0.261973 | \n", "
5 | \n", "-0.924046 | \n", "16.110458 | \n", "-1.786836 | \n", "
6 | \n", "-0.909357 | \n", "18.208616 | \n", "-1.753474 | \n", "
7 | \n", "-1.523700 | \n", "17.888836 | \n", "-0.262022 | \n", "
8 | \n", "-1.700855 | \n", "18.595100 | \n", "0.072298 | \n", "
9 | \n", "-1.490176 | \n", "22.120970 | \n", "0.063725 | \n", "
10 | \n", "-1.216591 | \n", "21.383211 | \n", "-0.708944 | \n", "
11 | \n", "-0.014494 | \n", "21.787636 | \n", "-0.402417 | \n", "
12 | \n", "0.625458 | \n", "21.205883 | \n", "-0.272478 | \n", "
13 | \n", "0.305759 | \n", "19.265335 | \n", "-0.480082 | \n", "
14 | \n", "-0.172271 | \n", "20.583281 | \n", "-0.580657 | \n", "
15 | \n", "-0.158516 | \n", "16.410503 | \n", "-2.263122 | \n", "
16 | \n", "-0.485935 | \n", "22.250452 | \n", "-0.292392 | \n", "
17 | \n", "-0.453527 | \n", "18.530380 | \n", "-0.677478 | \n", "
18 | \n", "-0.382364 | \n", "20.347807 | \n", "0.334156 | \n", "
19 | \n", "-0.688242 | \n", "20.049438 | \n", "0.306114 | \n", "
20 | \n", "0.466396 | \n", "15.867884 | \n", "1.843608 | \n", "
21 | \n", "0.355994 | \n", "20.886177 | \n", "1.915584 | \n", "
22 | \n", "0.193510 | \n", "18.926830 | \n", "2.335595 | \n", "
23 | \n", "0.128056 | \n", "20.565500 | \n", "1.859136 | \n", "
24 | \n", "0.770014 | \n", "19.866911 | \n", "2.293251 | \n", "
25 | \n", "0.928932 | \n", "19.648052 | \n", "1.856291 | \n", "
26 | \n", "1.197047 | \n", "16.213640 | \n", "1.365838 | \n", "
27 | \n", "1.145283 | \n", "23.506783 | \n", "1.379037 | \n", "
28 | \n", "1.186768 | \n", "22.549137 | \n", "1.419448 | \n", "
29 | \n", "0.579447 | \n", "19.437201 | \n", "2.397512 | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "
18286676 | \n", "2.344119 | \n", "27.980766 | \n", "-2.372994 | \n", "
18286677 | \n", "2.424500 | \n", "27.769838 | \n", "-2.445686 | \n", "
18286678 | \n", "2.328301 | \n", "26.979740 | \n", "-2.474418 | \n", "
18286679 | \n", "2.362773 | \n", "28.483915 | \n", "-2.440804 | \n", "
18286680 | \n", "2.466642 | \n", "28.569492 | \n", "-2.455950 | \n", "
18286681 | \n", "2.349971 | \n", "27.639442 | \n", "-2.430140 | \n", "
18286682 | \n", "2.386925 | \n", "29.215738 | \n", "-2.441397 | \n", "
18286683 | \n", "2.405529 | \n", "29.051693 | \n", "-2.445164 | \n", "
18286684 | \n", "2.416599 | \n", "27.439442 | \n", "-2.445778 | \n", "
18286685 | \n", "2.339918 | \n", "26.979265 | \n", "-2.393658 | \n", "
18286686 | \n", "2.387848 | \n", "28.929337 | \n", "-2.446784 | \n", "
18286687 | \n", "2.477501 | \n", "28.996275 | \n", "-2.343966 | \n", "
18286688 | \n", "2.491531 | \n", "27.439096 | \n", "-2.427254 | \n", "
18286689 | \n", "2.313082 | \n", "31.250610 | \n", "-2.341767 | \n", "
18286690 | \n", "2.474660 | \n", "28.234516 | \n", "-2.308086 | \n", "
18286691 | \n", "2.339188 | \n", "27.606724 | \n", "-2.334271 | \n", "
18286692 | \n", "2.423793 | \n", "27.666565 | \n", "-2.383434 | \n", "
18286693 | \n", "2.301899 | \n", "28.936501 | \n", "-2.395124 | \n", "
18286694 | \n", "2.349056 | \n", "30.725111 | \n", "-2.349983 | \n", "
18286695 | \n", "2.468506 | \n", "29.804134 | \n", "-2.473691 | \n", "
18286696 | \n", "2.468706 | \n", "27.269358 | \n", "-2.457757 | \n", "
18286697 | \n", "2.427693 | \n", "31.805853 | \n", "-2.362838 | \n", "
18286698 | \n", "2.420603 | \n", "31.342039 | \n", "-2.327386 | \n", "
18286699 | \n", "2.443260 | \n", "30.946449 | \n", "-2.300876 | \n", "
18286700 | \n", "2.442581 | \n", "32.081322 | \n", "-2.496407 | \n", "
18286701 | \n", "2.312973 | \n", "28.347523 | \n", "-2.426726 | \n", "
18286702 | \n", "2.447692 | \n", "26.709282 | \n", "-2.302419 | \n", "
18286703 | \n", "2.437479 | \n", "26.761944 | \n", "-2.390750 | \n", "
18286704 | \n", "2.494004 | \n", "33.038513 | \n", "-2.466166 | \n", "
18286705 | \n", "2.373600 | \n", "28.942278 | \n", "-2.316337 | \n", "
18286706 rows × 3 columns
\n", "mag_u_lsst | dec | ra |
---|---|---|
float32 | float32 | float32 |
21.2349 | -1.78091 | -0.213268 |
21.8391 | -2.06204 | -1.29391 |
21.0237 | -1.48434 | 0.681523 |
17.6038 | -0.92326 | -0.304187 |
19.3148 | -0.874825 | -0.261973 |
16.1105 | -0.924046 | -1.78684 |
18.2086 | -0.909357 | -1.75347 |
17.8888 | -1.5237 | -0.262022 |
18.5951 | -1.70086 | 0.0722983 |
22.121 | -1.49018 | 0.063725 |
... | ... | ... |
27.2694 | 2.46871 | -2.45776 |
31.8059 | 2.42769 | -2.36284 |
31.342 | 2.4206 | -2.32739 |
30.9464 | 2.44326 | -2.30088 |
32.0813 | 2.44258 | -2.49641 |
28.3475 | 2.31297 | -2.42673 |
26.7093 | 2.44769 | -2.30242 |
26.7619 | 2.43748 | -2.39075 |
33.0385 | 2.494 | -2.46617 |
28.9423 | 2.3736 | -2.31634 |