{ "cells": [ { "cell_type": "markdown", "id": "baking-battle", "metadata": {}, "source": [ "# PSA Utilities demo notebook\n", "\n", "This notebook demonstrates some of the functions of the [github.com/msbentley/psa_utils](github.com/msbentley/psa_utils) package.\n", "\n", "Please raise any bugs or criticisms as [GitHub issues](https://github.com/msbentley/psa_utils/issues)\n", "\n" ] }, { "cell_type": "code", "execution_count": 1, "id": "abroad-dining", "metadata": {}, "outputs": [], "source": [ "from psa_utils import download, tap, pdap, packager, geogen, internal" ] }, { "cell_type": "markdown", "id": "brazilian-possibility", "metadata": {}, "source": [ "# pdap\n", "\n", "This modules uses `requests` to make simple calls to a PDAP service. By default it uses the PSA unless another URL is given:" ] }, { "cell_type": "code", "execution_count": 2, "id": "distant-incident", "metadata": {}, "outputs": [], "source": [ "p = pdap.Pdap()" ] }, { "cell_type": "markdown", "id": "modular-kidney", "metadata": {}, "source": [ "The first function uses the meta-data endpoint to lists available datasets:\n", "\n", "## `get_datasets`\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "visible-internship", "metadata": {}, "outputs": [], "source": [ "dsets = p.get_datasets()" ] }, { "cell_type": "code", "execution_count": 4, "id": "serial-parallel", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DATA_SET.DATA_SET_IDDATA_SET.DATA_SET_NAMEDATA_SET.DATA_ACCESS_REFERENCEDATA_SET.XML_DESCRIPTIONDATA_SET.PRODUCER.FULL_NAMEDATA_SET.PRODUCER.INSTITUTION_NAMEDATA_SET.PRODUCER.NODE_NAMEDATA_SET.START_TIMEDATA_SET.STOP_TIMEDATA_SET.NPRODUCTSDATA_SET.MISSION_NAMEDATA_SET.INSTRUMENT_IDDATA_SET.INSTRUMENT_NAMEDATA_SET.TARGET_NAMERESOURCE_CLASSDATA_SET.RELEASE_DATE
0AIRUB-C-PHOTOCAM-2-EDR-HALLEY-1986-V1.0AIRUB-HALLEY-PHOTOGRAPHIC-PROJECT-EDR-1986-V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='AIR...1986-02-17T06:44:001986-04-17T09:13:001833EARTH300PENTACON-OPTICS-F4-300MM1P/HALLEYDATA_SET2006-03-01
1CH1ORB-L-C1XS-2-NPO-EDR-V1.0CHANDRAYAAN-1-ORBITER MOON C1XS 2 NPO EDR V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='CH1...2008-10-22T03:37:352009-08-28T18:21:3111738CHANDRAYAAN-1C1XSC1XSMOONDATA_SET2019-07-27
2CH1ORB-L-C1XS-4-NPO-REFDR-V1.0CHANDRAYAAN-1-ORBITER MOON C1XS 4 NPO REFDR V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='CH1...2008-11-20T18:32:11.3582009-08-28T18:21:01.5051675CHANDRAYAAN-1C1XSC1XSMOONDATA_SET2019-07-27
3CH1ORB-L-SARA-2-NPO-EDR-CENA-V1.0CHANDRAYAAN-1-ORBITER MOON SARA 2 NPO EDR CENA...http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='CH1...2008-12-08T11:54:53.3742009-08-12T17:02:49.417835CHANDRAYAAN-1SARASARAMOONDATA_SET2019-11-26
4CH1ORB-L-SARA-2-NPO-EDR-SWIM-V1.0CHANDRAYAAN-1-ORBITER MOON SARA 2 NPO EDR SWIM...http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='CH1...2008-12-08T13:53:52.1172009-08-12T17:02:49.3191424CHANDRAYAAN-1SARASARAMOONDATA_SET2019-12-05
\n", "
" ], "text/plain": [ " DATA_SET.DATA_SET_ID \\\n", "0 AIRUB-C-PHOTOCAM-2-EDR-HALLEY-1986-V1.0 \n", "1 CH1ORB-L-C1XS-2-NPO-EDR-V1.0 \n", "2 CH1ORB-L-C1XS-4-NPO-REFDR-V1.0 \n", "3 CH1ORB-L-SARA-2-NPO-EDR-CENA-V1.0 \n", "4 CH1ORB-L-SARA-2-NPO-EDR-SWIM-V1.0 \n", "\n", " DATA_SET.DATA_SET_NAME \\\n", "0 AIRUB-HALLEY-PHOTOGRAPHIC-PROJECT-EDR-1986-V1.0 \n", "1 CHANDRAYAAN-1-ORBITER MOON C1XS 2 NPO EDR V1.0 \n", "2 CHANDRAYAAN-1-ORBITER MOON C1XS 4 NPO REFDR V1.0 \n", "3 CHANDRAYAAN-1-ORBITER MOON SARA 2 NPO EDR CENA... \n", "4 CHANDRAYAAN-1-ORBITER MOON SARA 2 NPO EDR SWIM... \n", "\n", " DATA_SET.DATA_ACCESS_REFERENCE \\\n", "0 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "1 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "2 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "3 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "4 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "\n", " DATA_SET.XML_DESCRIPTION \\\n", "0 http://psa.esa.int/pdap/files?DATA_SET_ID='AIR... \n", "1 http://psa.esa.int/pdap/files?DATA_SET_ID='CH1... \n", "2 http://psa.esa.int/pdap/files?DATA_SET_ID='CH1... \n", "3 http://psa.esa.int/pdap/files?DATA_SET_ID='CH1... \n", "4 http://psa.esa.int/pdap/files?DATA_SET_ID='CH1... \n", "\n", " DATA_SET.PRODUCER.FULL_NAME DATA_SET.PRODUCER.INSTITUTION_NAME \\\n", "0 \n", "1 \n", "2 \n", "3 \n", "4 \n", "\n", " DATA_SET.PRODUCER.NODE_NAME DATA_SET.START_TIME \\\n", "0 1986-02-17T06:44:00 \n", "1 2008-10-22T03:37:35 \n", "2 2008-11-20T18:32:11.358 \n", "3 2008-12-08T11:54:53.374 \n", "4 2008-12-08T13:53:52.117 \n", "\n", " DATA_SET.STOP_TIME DATA_SET.NPRODUCTS DATA_SET.MISSION_NAME \\\n", "0 1986-04-17T09:13:00 1833 EARTH \n", "1 2009-08-28T18:21:31 11738 CHANDRAYAAN-1 \n", "2 2009-08-28T18:21:01.505 1675 CHANDRAYAAN-1 \n", "3 2009-08-12T17:02:49.417 835 CHANDRAYAAN-1 \n", "4 2009-08-12T17:02:49.319 1424 CHANDRAYAAN-1 \n", "\n", " DATA_SET.INSTRUMENT_ID DATA_SET.INSTRUMENT_NAME DATA_SET.TARGET_NAME \\\n", "0 300 PENTACON-OPTICS-F4-300MM 1P/HALLEY \n", "1 C1XS C1XS MOON \n", "2 C1XS C1XS MOON \n", "3 SARA SARA MOON \n", "4 SARA SARA MOON \n", "\n", " RESOURCE_CLASS DATA_SET.RELEASE_DATE \n", "0 DATA_SET 2006-03-01 \n", "1 DATA_SET 2019-07-27 \n", "2 DATA_SET 2019-07-27 \n", "3 DATA_SET 2019-11-26 \n", "4 DATA_SET 2019-12-05 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dsets.head()" ] }, { "cell_type": "markdown", "id": "square-springer", "metadata": {}, "source": [ "We have ALL of the datasets now, and their basic meta data, which we can easily slice and dice.\n", "\n", "For example, let's find those with Mars as a target:" ] }, { "cell_type": "code", "execution_count": 5, "id": "cultural-personality", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DATA_SET.DATA_SET_IDDATA_SET.DATA_SET_NAMEDATA_SET.DATA_ACCESS_REFERENCEDATA_SET.XML_DESCRIPTIONDATA_SET.PRODUCER.FULL_NAMEDATA_SET.PRODUCER.INSTITUTION_NAMEDATA_SET.PRODUCER.NODE_NAMEDATA_SET.START_TIMEDATA_SET.STOP_TIMEDATA_SET.NPRODUCTSDATA_SET.MISSION_NAMEDATA_SET.INSTRUMENT_IDDATA_SET.INSTRUMENT_NAMEDATA_SET.TARGET_NAMERESOURCE_CLASSDATA_SET.RELEASE_DATE
34MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT1-V1.0MARS EXPRESS ASPERA-3 RAW-CAL NTRL PARTICLE IM...http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='MEX...2006-01-01T22:42:28.0962007-10-01T01:42:54.521532MARS EXPRESSASPERA-3ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (...MARSDATA_SET2007-01-31
35MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT2-V1.0MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT2-V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='MEX...2007-10-01T04:39:48.772009-12-31T22:38:25.9882302MARS EXPRESSASPERA-3ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (...MARSDATA_SET2007-01-31
36MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT3-V1.0MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT3-V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='MEX...2010-01-01T15:49:39.1922012-12-31T22:26:30.6312026MARS EXPRESSASPERA-3ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (...MARSDATA_SET2011-01-19
37MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT4-V1.0MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT4-V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='MEX...2013-01-01T01:34:28.7072015-01-01T03:40:04.8231948MARS EXPRESSASPERA-3ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (...MARSDATA_SET2017-02-07
38MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT5-V1.0MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT5-V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='MEX...2015-01-01T04:00:26.6112017-01-01T02:45:17.4421465MARS EXPRESSASPERA-3ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (...MARSDATA_SET2017-02-07
...................................................
7080RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='RO-...2006-07-29T00:00:27.7412007-05-28T23:57:32.769304INTERNATIONAL ROSETTA MISSIONN/ASTANDARD RADIATION ENVIROMENT MONITORMARSDATA_SET2020-12-08
7129RO-X-SREM-5-MARS-V1.0ROSETTA-ORBITER X SREM 5 MARS V1.0http://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='RO-...2006-07-29T00:00:27.7412007-05-28T23:57:32.769303INTERNATIONAL ROSETTA MISSIONN/ASTANDARD RADIATION ENVIROMENT MONITORMARSDATA_SET2020-12-08
7177urn:esa:psa:em16_tgo_acsACShttp://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='urn...2016-03-14T00:00:002021-03-07T20:06:09.6142281756ExoMars 2016ACSACSurn:nasa:pds:context:target:planet.marsDATA_SET2020-07-31
7178urn:esa:psa:em16_tgo_casInstrument CASSIShttp://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='urn...2016-03-14T00:00:002021-03-07T21:44:08.4254166552ExoMars 2016CaSSISCASSISurn:nasa:pds:context:target:planet.marsDATA_SET2020-04-30
7180urn:esa:psa:em16_tgo_nmdNOMADhttp://psa.esa.int/pdap/download?RESOURCE_CLAS...http://psa.esa.int/pdap/files?DATA_SET_ID='urn...2016-04-04T06:00:002021-03-07T23:10:23.793149874ExoMars 2016NOMADNOMADurn:nasa:pds:context:target:planet.marsDATA_SET2020-02-29
\n", "

2751 rows × 16 columns

\n", "
" ], "text/plain": [ " DATA_SET.DATA_SET_ID \\\n", "34 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT1-V1.0 \n", "35 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT2-V1.0 \n", "36 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT3-V1.0 \n", "37 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT4-V1.0 \n", "38 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT5-V1.0 \n", "... ... \n", "7080 RO-X-SREM-2-MARS-V1.0 \n", "7129 RO-X-SREM-5-MARS-V1.0 \n", "7177 urn:esa:psa:em16_tgo_acs \n", "7178 urn:esa:psa:em16_tgo_cas \n", "7180 urn:esa:psa:em16_tgo_nmd \n", "\n", " DATA_SET.DATA_SET_NAME \\\n", "34 MARS EXPRESS ASPERA-3 RAW-CAL NTRL PARTICLE IM... \n", "35 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT2-V1.0 \n", "36 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT3-V1.0 \n", "37 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT4-V1.0 \n", "38 MEX-M-ASPERA3-2/3-EDR/RDR-NPI-EXT5-V1.0 \n", "... ... \n", "7080 ROSETTA-ORBITER X SREM 2 MARS V1.0 \n", "7129 ROSETTA-ORBITER X SREM 5 MARS V1.0 \n", "7177 ACS \n", "7178 Instrument CASSIS \n", "7180 NOMAD \n", "\n", " DATA_SET.DATA_ACCESS_REFERENCE \\\n", "34 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "35 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "36 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "37 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "38 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "... ... \n", "7080 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "7129 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "7177 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "7178 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "7180 http://psa.esa.int/pdap/download?RESOURCE_CLAS... \n", "\n", " DATA_SET.XML_DESCRIPTION \\\n", "34 http://psa.esa.int/pdap/files?DATA_SET_ID='MEX... \n", "35 http://psa.esa.int/pdap/files?DATA_SET_ID='MEX... \n", "36 http://psa.esa.int/pdap/files?DATA_SET_ID='MEX... \n", "37 http://psa.esa.int/pdap/files?DATA_SET_ID='MEX... \n", "38 http://psa.esa.int/pdap/files?DATA_SET_ID='MEX... \n", "... ... \n", "7080 http://psa.esa.int/pdap/files?DATA_SET_ID='RO-... \n", "7129 http://psa.esa.int/pdap/files?DATA_SET_ID='RO-... \n", "7177 http://psa.esa.int/pdap/files?DATA_SET_ID='urn... \n", "7178 http://psa.esa.int/pdap/files?DATA_SET_ID='urn... \n", "7180 http://psa.esa.int/pdap/files?DATA_SET_ID='urn... \n", "\n", " DATA_SET.PRODUCER.FULL_NAME DATA_SET.PRODUCER.INSTITUTION_NAME \\\n", "34 \n", "35 \n", "36 \n", "37 \n", "38 \n", "... ... ... \n", "7080 \n", "7129 \n", "7177 \n", "7178 \n", "7180 \n", "\n", " DATA_SET.PRODUCER.NODE_NAME DATA_SET.START_TIME \\\n", "34 2006-01-01T22:42:28.096 \n", "35 2007-10-01T04:39:48.77 \n", "36 2010-01-01T15:49:39.192 \n", "37 2013-01-01T01:34:28.707 \n", "38 2015-01-01T04:00:26.611 \n", "... ... ... \n", "7080 2006-07-29T00:00:27.741 \n", "7129 2006-07-29T00:00:27.741 \n", "7177 2016-03-14T00:00:00 \n", "7178 2016-03-14T00:00:00 \n", "7180 2016-04-04T06:00:00 \n", "\n", " DATA_SET.STOP_TIME DATA_SET.NPRODUCTS \\\n", "34 2007-10-01T01:42:54.52 1532 \n", "35 2009-12-31T22:38:25.988 2302 \n", "36 2012-12-31T22:26:30.631 2026 \n", "37 2015-01-01T03:40:04.823 1948 \n", "38 2017-01-01T02:45:17.442 1465 \n", "... ... ... \n", "7080 2007-05-28T23:57:32.769 304 \n", "7129 2007-05-28T23:57:32.769 303 \n", "7177 2021-03-07T20:06:09.614 2281756 \n", "7178 2021-03-07T21:44:08.425 4166552 \n", "7180 2021-03-07T23:10:23.793 149874 \n", "\n", " DATA_SET.MISSION_NAME DATA_SET.INSTRUMENT_ID \\\n", "34 MARS EXPRESS ASPERA-3 \n", "35 MARS EXPRESS ASPERA-3 \n", "36 MARS EXPRESS ASPERA-3 \n", "37 MARS EXPRESS ASPERA-3 \n", "38 MARS EXPRESS ASPERA-3 \n", "... ... ... \n", "7080 INTERNATIONAL ROSETTA MISSION N/A \n", "7129 INTERNATIONAL ROSETTA MISSION N/A \n", "7177 ExoMars 2016 ACS \n", "7178 ExoMars 2016 CaSSIS \n", "7180 ExoMars 2016 NOMAD \n", "\n", " DATA_SET.INSTRUMENT_NAME \\\n", "34 ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (... \n", "35 ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (... \n", "36 ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (... \n", "37 ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (... \n", "38 ANALYZER OF SPACE PLASMA AND ENERGETIC ATOMS (... \n", "... ... \n", "7080 STANDARD RADIATION ENVIROMENT MONITOR \n", "7129 STANDARD RADIATION ENVIROMENT MONITOR \n", "7177 ACS \n", "7178 CASSIS \n", "7180 NOMAD \n", "\n", " DATA_SET.TARGET_NAME RESOURCE_CLASS \\\n", "34 MARS DATA_SET \n", "35 MARS DATA_SET \n", "36 MARS DATA_SET \n", "37 MARS DATA_SET \n", "38 MARS DATA_SET \n", "... ... ... \n", "7080 MARS DATA_SET \n", "7129 MARS DATA_SET \n", "7177 urn:nasa:pds:context:target:planet.mars DATA_SET \n", "7178 urn:nasa:pds:context:target:planet.mars DATA_SET \n", "7180 urn:nasa:pds:context:target:planet.mars DATA_SET \n", "\n", " DATA_SET.RELEASE_DATE \n", "34 2007-01-31 \n", "35 2007-01-31 \n", "36 2011-01-19 \n", "37 2017-02-07 \n", "38 2017-02-07 \n", "... ... \n", "7080 2020-12-08 \n", "7129 2020-12-08 \n", "7177 2020-07-31 \n", "7178 2020-04-30 \n", "7180 2020-02-29 \n", "\n", "[2751 rows x 16 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dsets[dsets['DATA_SET.TARGET_NAME'].str.contains('mars', case=False)]" ] }, { "cell_type": "markdown", "id": "trained-upset", "metadata": {}, "source": [ "## `get_products`" ] }, { "cell_type": "code", "execution_count": 6, "id": "afraid-shareware", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_products\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m\n", "Queries the meta-data endpoint for products in the dataset ID\n", "given in the call\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/pdap.py\n", "\u001b[0;31mType:\u001b[0m method\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "p.get_products?" ] }, { "cell_type": "code", "execution_count": 7, "id": "interpreted-episode", "metadata": {}, "outputs": [], "source": [ "products = p.get_products(dataset_id='RO-X-SREM-2-MARS-V1.0')" ] }, { "cell_type": "code", "execution_count": 8, "id": "happy-jimmy", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
PRODUCT.PRODUCT_IDPRODUCT.DATA_ACCESS_REFERENCEDATA_SET.DATA_SET_IDDATA_SET.DATA_SET_NAMEDATA_SET.MISSION_NAMEDATA_SET.PRODUCER.FULL_NAMEDATA_SET.PRODUCER.INSTITUTION_NAMEDATA_SET.PRODUCER.NODE_NAMEPRODUCT.TARGET_NAMEPRODUCT.TARGET_TYPEPRODUCT.INSTRUMENT_IDPRODUCT.INSTRUMENT_NAMEPRODUCT.START_TIMEPRODUCT.STOP_TIMEPRODUCT.ICON_ACCESS_REFERENCERESOURCE_CLASSVID
0RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070528http://psa.esa.int/pdap/download?RESOURCE_CLAS...RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0INTERNATIONAL ROSETTA MISSIONMARSPLANETN/ASTANDARD RADIATION ENVIROMENT MONITOR2007-05-28T00:03:03.7552007-05-28T23:57:32.769PRODUCT1.0
1RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070527http://psa.esa.int/pdap/download?RESOURCE_CLAS...RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0INTERNATIONAL ROSETTA MISSIONMARSPLANETN/ASTANDARD RADIATION ENVIROMENT MONITOR2007-05-27T00:00:04.242007-05-27T23:59:04.255PRODUCT1.0
2RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070526http://psa.esa.int/pdap/download?RESOURCE_CLAS...RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0INTERNATIONAL ROSETTA MISSIONMARSPLANETN/ASTANDARD RADIATION ENVIROMENT MONITOR2007-05-26T00:00:28.2252007-05-26T23:56:03.74PRODUCT1.0
3RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070525http://psa.esa.int/pdap/download?RESOURCE_CLAS...RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0INTERNATIONAL ROSETTA MISSIONMARSPLANETN/ASTANDARD RADIATION ENVIROMENT MONITOR2007-05-25T00:00:00.2112007-05-25T23:56:32.225PRODUCT1.0
4RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070524http://psa.esa.int/pdap/download?RESOURCE_CLAS...RO-X-SREM-2-MARS-V1.0ROSETTA-ORBITER X SREM 2 MARS V1.0INTERNATIONAL ROSETTA MISSIONMARSPLANETN/ASTANDARD RADIATION ENVIROMENT MONITOR2007-05-24T00:02:32.6962007-05-24T23:56:00.211PRODUCT1.0
\n", "
" ], "text/plain": [ " PRODUCT.PRODUCT_ID \\\n", "0 RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070528 \n", "1 RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070527 \n", "2 RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070526 \n", "3 RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070525 \n", "4 RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070524 \n", "\n", " PRODUCT.DATA_ACCESS_REFERENCE DATA_SET.DATA_SET_ID \\\n", "0 http://psa.esa.int/pdap/download?RESOURCE_CLAS... RO-X-SREM-2-MARS-V1.0 \n", "1 http://psa.esa.int/pdap/download?RESOURCE_CLAS... RO-X-SREM-2-MARS-V1.0 \n", "2 http://psa.esa.int/pdap/download?RESOURCE_CLAS... RO-X-SREM-2-MARS-V1.0 \n", "3 http://psa.esa.int/pdap/download?RESOURCE_CLAS... RO-X-SREM-2-MARS-V1.0 \n", "4 http://psa.esa.int/pdap/download?RESOURCE_CLAS... RO-X-SREM-2-MARS-V1.0 \n", "\n", " DATA_SET.DATA_SET_NAME DATA_SET.MISSION_NAME \\\n", "0 ROSETTA-ORBITER X SREM 2 MARS V1.0 INTERNATIONAL ROSETTA MISSION \n", "1 ROSETTA-ORBITER X SREM 2 MARS V1.0 INTERNATIONAL ROSETTA MISSION \n", "2 ROSETTA-ORBITER X SREM 2 MARS V1.0 INTERNATIONAL ROSETTA MISSION \n", "3 ROSETTA-ORBITER X SREM 2 MARS V1.0 INTERNATIONAL ROSETTA MISSION \n", "4 ROSETTA-ORBITER X SREM 2 MARS V1.0 INTERNATIONAL ROSETTA MISSION \n", "\n", " DATA_SET.PRODUCER.FULL_NAME DATA_SET.PRODUCER.INSTITUTION_NAME \\\n", "0 \n", "1 \n", "2 \n", "3 \n", "4 \n", "\n", " DATA_SET.PRODUCER.NODE_NAME PRODUCT.TARGET_NAME PRODUCT.TARGET_TYPE \\\n", "0 MARS PLANET \n", "1 MARS PLANET \n", "2 MARS PLANET \n", "3 MARS PLANET \n", "4 MARS PLANET \n", "\n", " PRODUCT.INSTRUMENT_ID PRODUCT.INSTRUMENT_NAME \\\n", "0 N/A STANDARD RADIATION ENVIROMENT MONITOR \n", "1 N/A STANDARD RADIATION ENVIROMENT MONITOR \n", "2 N/A STANDARD RADIATION ENVIROMENT MONITOR \n", "3 N/A STANDARD RADIATION ENVIROMENT MONITOR \n", "4 N/A STANDARD RADIATION ENVIROMENT MONITOR \n", "\n", " PRODUCT.START_TIME PRODUCT.STOP_TIME \\\n", "0 2007-05-28T00:03:03.755 2007-05-28T23:57:32.769 \n", "1 2007-05-27T00:00:04.24 2007-05-27T23:59:04.255 \n", "2 2007-05-26T00:00:28.225 2007-05-26T23:56:03.74 \n", "3 2007-05-25T00:00:00.211 2007-05-25T23:56:32.225 \n", "4 2007-05-24T00:02:32.696 2007-05-24T23:56:00.211 \n", "\n", " PRODUCT.ICON_ACCESS_REFERENCE RESOURCE_CLASS VID \n", "0 PRODUCT 1.0 \n", "1 PRODUCT 1.0 \n", "2 PRODUCT 1.0 \n", "3 PRODUCT 1.0 \n", "4 PRODUCT 1.0 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "products.head()" ] }, { "cell_type": "markdown", "id": "silent-central", "metadata": {}, "source": [ "Let's look at what one entry contains:" ] }, { "cell_type": "code", "execution_count": 9, "id": "electronic-ontario", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "PRODUCT.PRODUCT_ID RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070528\n", "PRODUCT.DATA_ACCESS_REFERENCE http://psa.esa.int/pdap/download?RESOURCE_CLAS...\n", "DATA_SET.DATA_SET_ID RO-X-SREM-2-MARS-V1.0\n", "DATA_SET.DATA_SET_NAME ROSETTA-ORBITER X SREM 2 MARS V1.0\n", "DATA_SET.MISSION_NAME INTERNATIONAL ROSETTA MISSION\n", "DATA_SET.PRODUCER.FULL_NAME \n", "DATA_SET.PRODUCER.INSTITUTION_NAME \n", "DATA_SET.PRODUCER.NODE_NAME \n", "PRODUCT.TARGET_NAME MARS\n", "PRODUCT.TARGET_TYPE PLANET\n", "PRODUCT.INSTRUMENT_ID N/A\n", "PRODUCT.INSTRUMENT_NAME STANDARD RADIATION ENVIROMENT MONITOR\n", "PRODUCT.START_TIME 2007-05-28T00:03:03.755\n", "PRODUCT.STOP_TIME 2007-05-28T23:57:32.769\n", "PRODUCT.ICON_ACCESS_REFERENCE \n", "RESOURCE_CLASS PRODUCT\n", "VID 1.0\n", "Name: 0, dtype: object" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "products.iloc[0]" ] }, { "cell_type": "markdown", "id": "acoustic-nightmare", "metadata": {}, "source": [ "## `get_product`" ] }, { "cell_type": "code", "execution_count": 10, "id": "after-madonna", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_product\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproduct_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m \n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/pdap.py\n", "\u001b[0;31mType:\u001b[0m method\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "p.get_product?" ] }, { "cell_type": "code", "execution_count": 11, "id": "periodic-radio", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "PRODUCT.PRODUCT_ID RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070528\n", "PRODUCT.DATA_ACCESS_REFERENCE http://psa.esa.int/pdap/download?RESOURCE_CLAS...\n", "DATA_SET.DATA_SET_ID RO-X-SREM-2-MARS-V1.0\n", "DATA_SET.DATA_SET_NAME ROSETTA-ORBITER X SREM 2 MARS V1.0\n", "DATA_SET.MISSION_NAME INTERNATIONAL ROSETTA MISSION\n", "DATA_SET.PRODUCER.FULL_NAME \n", "DATA_SET.PRODUCER.INSTITUTION_NAME \n", "DATA_SET.PRODUCER.NODE_NAME \n", "PRODUCT.TARGET_NAME MARS\n", "PRODUCT.TARGET_TYPE PLANET\n", "PRODUCT.INSTRUMENT_ID N/A\n", "PRODUCT.INSTRUMENT_NAME STANDARD RADIATION ENVIROMENT MONITOR\n", "PRODUCT.START_TIME 2007-05-28T00:03:03.755\n", "PRODUCT.STOP_TIME 2007-05-28T23:57:32.769\n", "PRODUCT.ICON_ACCESS_REFERENCE \n", "RESOURCE_CLASS PRODUCT\n", "VID 1.0\n", "Name: 0, dtype: object" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "p.get_product('RO-X-SREM-2-MARS-V1.0:DATA:SREM_L2_20070528')" ] }, { "cell_type": "markdown", "id": "legislative-cable", "metadata": {}, "source": [ "## `get_files`\n", "\n", "Uses the files endpoint to retrieve a list of files in a given dataset." ] }, { "cell_type": "code", "execution_count": 12, "id": "considerable-employment", "metadata": {}, "outputs": [], "source": [ "files = p.get_files(dataset_id='RO-X-SREM-2-MARS-V1.0')" ] }, { "cell_type": "code", "execution_count": 13, "id": "molecular-reggae", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ReferenceDataSetIdProductIdRELATIVE_DIRECTORYFilename
0http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...RO-X-SREM-2-MARS-V1.0SREM_L2_20060918/DATA/SREM_L2_20060918.TAB
1http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...RO-X-SREM-2-MARS-V1.0SREM_L2_20061227/DATA/SREM_L2_20061227.TAB
2http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...RO-X-SREM-2-MARS-V1.0SREM_L2_20060926/DATA/SREM_L2_20060926.TAB
3http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...RO-X-SREM-2-MARS-V1.0/EXTRAS/SREM_ROSETTA_PACC_20070508.CDF
4http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...RO-X-SREM-2-MARS-V1.0/EXTRAS/SREM_ROSETTA_PACC_20070419.CDF
\n", "
" ], "text/plain": [ " Reference DataSetId \\\n", "0 http://psa.esa.int/pdap/fileaccess?ID=INTERNAT... RO-X-SREM-2-MARS-V1.0 \n", "1 http://psa.esa.int/pdap/fileaccess?ID=INTERNAT... RO-X-SREM-2-MARS-V1.0 \n", "2 http://psa.esa.int/pdap/fileaccess?ID=INTERNAT... RO-X-SREM-2-MARS-V1.0 \n", "3 http://psa.esa.int/pdap/fileaccess?ID=INTERNAT... RO-X-SREM-2-MARS-V1.0 \n", "4 http://psa.esa.int/pdap/fileaccess?ID=INTERNAT... RO-X-SREM-2-MARS-V1.0 \n", "\n", " ProductId RELATIVE_DIRECTORY Filename \n", "0 SREM_L2_20060918 /DATA/ SREM_L2_20060918.TAB \n", "1 SREM_L2_20061227 /DATA/ SREM_L2_20061227.TAB \n", "2 SREM_L2_20060926 /DATA/ SREM_L2_20060926.TAB \n", "3 /EXTRAS/ SREM_ROSETTA_PACC_20070508.CDF \n", "4 /EXTRAS/ SREM_ROSETTA_PACC_20070419.CDF " ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "files.head()" ] }, { "cell_type": "code", "execution_count": 14, "id": "fitted-belfast", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Reference http://psa.esa.int/pdap/fileaccess?ID=INTERNAT...\n", "DataSetId RO-X-SREM-2-MARS-V1.0\n", "ProductId SREM_L2_20060918\n", "RELATIVE_DIRECTORY /DATA/\n", "Filename SREM_L2_20060918.TAB\n", "Name: 0, dtype: object" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "files.iloc[0]" ] }, { "cell_type": "markdown", "id": "caroline-stage", "metadata": {}, "source": [ "# tap\n", "\n", "The `tap` module contains a single class `PsaTap` and various convenience functions that call this class. It is basically a very thin wrapper around `astroquery`'s TAP functionality." ] }, { "cell_type": "code", "execution_count": 15, "id": "shared-narrative", "metadata": {}, "outputs": [], "source": [ "psa = tap.PsaTap()" ] }, { "cell_type": "markdown", "id": "american-festival", "metadata": {}, "source": [ "Currently PsaTap includes only a single method, `query` which itself calls the `astroquery` Tap function and converts the returned data to a Pandas DataFrame." ] }, { "cell_type": "markdown", "id": "established-novel", "metadata": {}, "source": [ "## `query`" ] }, { "cell_type": "code", "execution_count": 16, "id": "together-prairie", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m\n", "\u001b[0mpsa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0msync\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mdropna\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mjob_wait_cycles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mjob_wait_time\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m Make a simple query and return the data as a pandas DataFrame\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/tap.py\n", "\u001b[0;31mType:\u001b[0m method\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "psa.query?" ] }, { "cell_type": "markdown", "id": "prostate-instruction", "metadata": {}, "source": [ "By default you can simply pass a query and this will run a synchronous query job and return. For queries that may return >2k results, set sync=False - you can adjust the number of wait cycles, and the time (in seconds) of each, before query aborts.\n", "\n", "Note also the dropna boolean - if True, any columns which are *all* NaN will be dropped from the returned DataFrame. This is useful because the epn_core schema contains a lot of fields which are not _yet_ populated in the PSA database." ] }, { "cell_type": "code", "execution_count": 17, "id": "hairy-performance", "metadata": {}, "outputs": [], "source": [ "top10 = psa.query('select top 10 * from epn_core')" ] }, { "cell_type": "code", "execution_count": 18, "id": "progressive-voluntary", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
access_estsizeaccess_formataccess_urlcreation_datedataproduct_typegranule_gidgranule_uidinstrument_host_nameinstrument_namemeasurement_type...processing_levelrelease_dateservice_titlespatial_frame_types_regiontarget_classtarget_namethumbnail_urltime_maxtime_min
024application/x-pds-ziphttps://archives.esac.esa.int/psa/pdap/downloa...2021-03-08T14:04:30.978734ciCH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATACH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...Chandrayaan-1C1XS...32019-07-27T00:00:00.0psanonesatelliteMoon2008-11-20 18:55:51.0000094722008-11-20 18:27:32.000011520
125application/x-pds-ziphttps://archives.esac.esa.int/psa/pdap/downloa...2021-03-08T14:04:30.978734ciCH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATACH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...Chandrayaan-1C1XS...32019-07-27T00:00:00.0psanonesatelliteMoon2008-11-23 00:24:13.9999992322008-11-22 23:38:54.999981568
224application/x-pds-ziphttps://archives.esac.esa.int/psa/pdap/downloa...2021-03-08T14:04:30.978734ciCH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATACH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...Chandrayaan-1C1XS...32019-07-27T00:00:00.0psanonesatelliteMoon2008-11-24 02:31:26.0000028162008-11-24 02:31:26.000002816
324application/x-pds-ziphttps://archives.esac.esa.int/psa/pdap/downloa...2021-03-08T14:04:30.978734ciCH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATACH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...Chandrayaan-1C1XS...32019-07-27T00:00:00.0psanonesatelliteMoon2008-11-24 04:00:05.9999943682008-11-24 04:00:05.999994368
424application/x-pds-ziphttps://archives.esac.esa.int/psa/pdap/downloa...2021-03-08T14:04:30.978734ciCH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATACH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...Chandrayaan-1C1XS...32019-07-27T00:00:00.0psanonesatelliteMoon2008-11-28 14:45:38.0000084482008-11-28 14:45:38.000008448
\n", "

5 rows × 22 columns

\n", "
" ], "text/plain": [ " access_estsize access_format \\\n", "0 24 application/x-pds-zip \n", "1 25 application/x-pds-zip \n", "2 24 application/x-pds-zip \n", "3 24 application/x-pds-zip \n", "4 24 application/x-pds-zip \n", "\n", " access_url \\\n", "0 https://archives.esac.esa.int/psa/pdap/downloa... \n", "1 https://archives.esac.esa.int/psa/pdap/downloa... \n", "2 https://archives.esac.esa.int/psa/pdap/downloa... \n", "3 https://archives.esac.esa.int/psa/pdap/downloa... \n", "4 https://archives.esac.esa.int/psa/pdap/downloa... \n", "\n", " creation_date dataproduct_type \\\n", "0 2021-03-08T14:04:30.978734 ci \n", "1 2021-03-08T14:04:30.978734 ci \n", "2 2021-03-08T14:04:30.978734 ci \n", "3 2021-03-08T14:04:30.978734 ci \n", "4 2021-03-08T14:04:30.978734 ci \n", "\n", " granule_gid \\\n", "0 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA \n", "1 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA \n", "2 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA \n", "3 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA \n", "4 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA \n", "\n", " granule_uid instrument_host_name \\\n", "0 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R... Chandrayaan-1 \n", "1 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R... Chandrayaan-1 \n", "2 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R... Chandrayaan-1 \n", "3 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R... Chandrayaan-1 \n", "4 CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R... Chandrayaan-1 \n", "\n", " instrument_name measurement_type ... processing_level \\\n", "0 C1XS ... 3 \n", "1 C1XS ... 3 \n", "2 C1XS ... 3 \n", "3 C1XS ... 3 \n", "4 C1XS ... 3 \n", "\n", " release_date service_title spatial_frame_type s_region \\\n", "0 2019-07-27T00:00:00.0 psa none \n", "1 2019-07-27T00:00:00.0 psa none \n", "2 2019-07-27T00:00:00.0 psa none \n", "3 2019-07-27T00:00:00.0 psa none \n", "4 2019-07-27T00:00:00.0 psa none \n", "\n", " target_class target_name thumbnail_url time_max \\\n", "0 satellite Moon 2008-11-20 18:55:51.000009472 \n", "1 satellite Moon 2008-11-23 00:24:13.999999232 \n", "2 satellite Moon 2008-11-24 02:31:26.000002816 \n", "3 satellite Moon 2008-11-24 04:00:05.999994368 \n", "4 satellite Moon 2008-11-28 14:45:38.000008448 \n", "\n", " time_min \n", "0 2008-11-20 18:27:32.000011520 \n", "1 2008-11-22 23:38:54.999981568 \n", "2 2008-11-24 02:31:26.000002816 \n", "3 2008-11-24 04:00:05.999994368 \n", "4 2008-11-28 14:45:38.000008448 \n", "\n", "[5 rows x 22 columns]" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "top10.head()" ] }, { "cell_type": "markdown", "id": "changing-pennsylvania", "metadata": {}, "source": [ "Let's look at one single entry:" ] }, { "cell_type": "code", "execution_count": 19, "id": "documentary-librarian", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "access_estsize 24\n", "access_format application/x-pds-zip\n", "access_url https://archives.esac.esa.int/psa/pdap/downloa...\n", "creation_date 2021-03-08T14:04:30.978734\n", "dataproduct_type ci\n", "granule_gid CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA\n", "granule_uid CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...\n", "instrument_host_name Chandrayaan-1\n", "instrument_name C1XS\n", "measurement_type \n", "modification_date 2021-03-08T14:04:30.978734\n", "obs_id CH1ORB-L-C1XS-2-NPO-EDR-V1.0:DATA:C1XS_NECAX_R...\n", "processing_level 3\n", "release_date 2019-07-27T00:00:00.0\n", "service_title psa\n", "spatial_frame_type none\n", "s_region \n", "target_class satellite\n", "target_name Moon\n", "thumbnail_url \n", "time_max 2008-11-20 18:55:51.000009472\n", "time_min 2008-11-20 18:27:32.000011520\n", "Name: 0, dtype: object" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "top10.iloc[0]" ] }, { "cell_type": "markdown", "id": "parallel-steel", "metadata": {}, "source": [ "Here note also that `time_min` and `time_max` have been converted from Julian dates to standard date/times" ] }, { "cell_type": "markdown", "id": "spread-monday", "metadata": {}, "source": [ "Now let's try exceeding that 2k limit, also increasing astroquery's verbosity:" ] }, { "cell_type": "code", "execution_count": 20, "id": "bright-giant", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Launched query: 'select TOP 2000 * from epn_core where instrument_name='OSIRIS''\n", "------>https\n", "host = archives.esac.esa.int:443\n", "context = /psa/epn-tap/tap//sync\n", "Content-type = application/x-www-form-urlencoded\n", "200 \n", "[('Date', 'Mon, 08 Mar 2021 17:46:50 GMT'), ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux)'), ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), ('Expires', '0'), ('X-XSS-Protection', '1; mode=block'), ('X-Frame-Options', 'DENY'), ('X-Content-Type-Options', 'nosniff'), ('Content-Type', 'application/x-votable+xml'), ('Set-Cookie', 'JSESSIONID=1C2A96F3B34D2B9EAF6F85EDAD9FCA3A;path=/psa/epn-tap;HttpOnly'), ('Vary', 'Accept-Encoding'), ('Transfer-Encoding', 'chunked')]\n", "Retrieving sync. results...\n", "Query finished.\n", "WARNING 2021-03-08 18:46:51 (psa_utils.tap): results incomplete due to synchronous query limit - repeat with sync=false\n" ] } ], "source": [ "size_test = psa.query(\"select * from epn_core where instrument_name='OSIRIS'\", verbose=True)" ] }, { "cell_type": "code", "execution_count": 21, "id": "changed-printing", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2000" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(size_test)" ] }, { "cell_type": "markdown", "id": "opposite-expression", "metadata": {}, "source": [ "So here we see that astroquery actually inserts an extra `TOP 2000` clause into the query statement. Note that if you specify `TOP` in your query, you automatically override this:" ] }, { "cell_type": "code", "execution_count": 22, "id": "announced-scholarship", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Launched query: 'select top 3000 * from epn_core where instrument_name='OSIRIS''\n", "------>https\n", "host = archives.esac.esa.int:443\n", "context = /psa/epn-tap/tap//sync\n", "Content-type = application/x-www-form-urlencoded\n", "200 \n", "[('Date', 'Mon, 08 Mar 2021 17:46:55 GMT'), ('Server', 'Apache/2.4.6 (Red Hat Enterprise Linux)'), ('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'), ('Pragma', 'no-cache'), ('Expires', '0'), ('X-XSS-Protection', '1; mode=block'), ('X-Frame-Options', 'DENY'), ('X-Content-Type-Options', 'nosniff'), ('Content-Type', 'application/x-votable+xml'), ('Set-Cookie', 'JSESSIONID=BC585CFC3BAFDB3CE865FD0271FC609F;path=/psa/epn-tap;HttpOnly'), ('Vary', 'Accept-Encoding'), ('Transfer-Encoding', 'chunked')]\n", "Retrieving sync. results...\n", "Query finished.\n" ] } ], "source": [ "size_test = psa.query(\"select top 3000 * from epn_core where instrument_name='OSIRIS'\", verbose=True)" ] }, { "cell_type": "code", "execution_count": 23, "id": "musical-newcastle", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3000" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(size_test)" ] }, { "cell_type": "markdown", "id": "mounted-maintenance", "metadata": {}, "source": [ "There is also the function, especially for larger queries, to run asynchronously:" ] }, { "cell_type": "code", "execution_count": 24, "id": "european-jason", "metadata": {}, "outputs": [], "source": [ "asyn = psa.query(\"select top 10000 * from epn_core where instrument_name='ACS'\", \n", " sync=False, job_wait_cycles=2, job_wait_time=10)" ] }, { "cell_type": "code", "execution_count": 25, "id": "medium-audience", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10000" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(asyn)" ] }, { "cell_type": "markdown", "id": "impressive-sugar", "metadata": {}, "source": [ "You can of course also perform queries that return values other than than a product list:" ] }, { "cell_type": "code", "execution_count": 26, "id": "robust-diagnosis", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
count_all
01715
\n", "
" ], "text/plain": [ " count_all\n", "0 1715" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "psa.query(\"select count(*) from epn_core where instrument_name='MCAM'\")" ] }, { "cell_type": "markdown", "id": "favorite-cocktail", "metadata": {}, "source": [ "Note that products have an access_url which will download the product - this is used in the `download` module." ] }, { "cell_type": "code", "execution_count": 27, "id": "fifth-construction", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
count_all
04087604
\n", "
" ], "text/plain": [ " count_all\n", "0 4087604" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "psa.query(\"select count(*) from epn_core where instrument_name='CaSSIS'\")" ] }, { "cell_type": "code", "execution_count": 28, "id": "american-disney", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
count_all
024435
\n", "
" ], "text/plain": [ " count_all\n", "0 24435" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "psa.query(\"select count(*) from epn_core where instrument_name='CaSSIS' and granule_uid like '%sti%'\")" ] }, { "cell_type": "code", "execution_count": 29, "id": "judicial-trust", "metadata": {}, "outputs": [], "source": [ "cassis = psa.query(\"select top 10 * from epn_core where instrument_name='CaSSIS' and granule_uid like '%sti%'\")" ] }, { "cell_type": "code", "execution_count": 30, "id": "available-tunnel", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "access_estsize 112442\n", "access_format application/x-pds-zip\n", "access_url https://archives.esac.esa.int/psa/pdap/downloa...\n", "creation_date 2021-03-08T14:04:30.978734\n", "dataproduct_type ci\n", "granule_gid urn:esa:psa:em16_tgo_cas:data_calibrated\n", "granule_uid urn:esa:psa:em16_tgo_cas:data_calibrated:cas_c...\n", "instrument_host_name ExoMars 2016\n", "instrument_name CaSSIS\n", "measurement_type \n", "modification_date 2021-03-08T14:04:30.978734\n", "obs_id urn:esa:psa:em16_tgo_cas:data_calibrated:cas_c...\n", "processing_level 3\n", "release_date 2018-10-14T00:00:00.0\n", "service_title psa\n", "spatial_frame_type none\n", "s_region \n", "target_class planet\n", "target_name Mars\n", "thumbnail_url https://archives.esac.esa.int/psa/pdap/fileacc...\n", "time_max 2018-04-14 16:15:30.000011520\n", "time_min 2018-04-14 16:15:16.000007168\n", "Name: 0, dtype: object" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cassis.iloc[0]" ] }, { "cell_type": "markdown", "id": "threatened-transsexual", "metadata": {}, "source": [ "So you can always use this `access_url` to download the data product - the `download` module has functions to do just this." ] }, { "cell_type": "markdown", "id": "textile-resolution", "metadata": {}, "source": [ "# download" ] }, { "cell_type": "markdown", "id": "conservative-fiction", "metadata": {}, "source": [ "## `download_by_query`\n", "This is the key function in this module - it accepts an ADQL query string, uses the `tap` module to run the query, downloads the referenced files (if they are public) and unzips them to the location of your choice." ] }, { "cell_type": "code", "execution_count": 31, "id": "laughing-gossip", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m \u001b[0mdownload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload_by_query\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munzip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtidy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m\n", "Runs a query against the PSA's EPN-TAP interface. Any products which match,\n", "and are public (have a download URL) will be downloaded and the zips placed\n", "into output_dir. If unzip=True they will be unzipped into output_dir and\n", "if tidy=True the zips will be removed after use\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/download.py\n", "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "download.download_by_query?" ] }, { "cell_type": "code", "execution_count": 32, "id": "authorized-reverse", "metadata": {}, "outputs": [], "source": [ "q = \"select top 2 * from epn_core where instrument_name='OSIRIS' and granule_uid like '%.FIT%'\"" ] }, { "cell_type": "code", "execution_count": 33, "id": "difficult-bermuda", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO 2021-03-08 18:47:36 (psa_utils.download): downloading product N20080814T023137516ID20F22.FIT\n", "INFO 2021-03-08 18:47:39 (psa_utils.download): downloaded file Download-20210308184736.zip\n", "INFO 2021-03-08 18:47:39 (psa_utils.download): downloading product N20080804T023124677ID20F22.FIT\n", "INFO 2021-03-08 18:47:40 (psa_utils.download): downloaded file Download-20210308184739.zip\n" ] } ], "source": [ "filelist = download.download_by_query(q, output_dir='/tmp')" ] }, { "cell_type": "code", "execution_count": 34, "id": "cosmetic-broadcasting", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['/tmp/RO-A-OSINAC-2-AST1-STEINSFLYBY-V2.0/DATA/FIT/N20080814T023137516ID20F22.LBL',\n", " '/tmp/RO-A-OSINAC-2-AST1-STEINSFLYBY-V2.0/DATA/FIT/N20080804T023124677ID20F22.LBL',\n", " '/tmp/RO-A-OSINAC-2-AST1-STEINSFLYBY-V2.0/DATA/FIT/N20080804T023124677ID20F22.FIT',\n", " '/tmp/inventory.txt',\n", " '/tmp/RO-A-OSINAC-2-AST1-STEINSFLYBY-V2.0/DATA/FIT/N20080814T023137516ID20F22.FIT']" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "filelist" ] }, { "cell_type": "markdown", "id": "computational-allergy", "metadata": {}, "source": [ "Note that in most cases these settings are what you want to retrieve products. If you prefer to keep the zips after extract, set tidy=False - you will still get the list of individual files returned. If you just want to download the zips, set unzip=False and you will get the list of zips returned:" ] }, { "cell_type": "code", "execution_count": 35, "id": "adverse-vault", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING 2021-03-08 18:47:40 (psa_utils.download): cannot remove source files without decompressiong - setting tidy=False\n", "INFO 2021-03-08 18:47:46 (psa_utils.download): downloading product N20080814T023137516ID20F22.FIT\n", "INFO 2021-03-08 18:47:48 (psa_utils.download): downloaded file Download-20210308184746.zip\n", "INFO 2021-03-08 18:47:48 (psa_utils.download): downloading product N20080804T023124677ID20F22.FIT\n", "INFO 2021-03-08 18:47:49 (psa_utils.download): downloaded file Download-20210308184748.zip\n" ] } ], "source": [ "filelist = download.download_by_query(q, output_dir='/tmp', unzip=False)" ] }, { "cell_type": "code", "execution_count": 36, "id": "seeing-pound", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['/tmp/Download-20210308184746.zip', '/tmp/Download-20210308184748.zip']" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "filelist" ] }, { "cell_type": "markdown", "id": "fifteen-rendering", "metadata": {}, "source": [ "There is a convenience function to download by the logical identifer, or product ID:" ] }, { "cell_type": "code", "execution_count": 37, "id": "broad-darkness", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m \u001b[0mdownload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload_by_lid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munzip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtidy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m \n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/download.py\n", "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "download.download_by_lid?" ] }, { "cell_type": "code", "execution_count": 38, "id": "logical-transformation", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO 2021-03-08 18:47:54 (psa_utils.download): downloading product N20080804T023124677ID20F22.FIT\n", "INFO 2021-03-08 18:47:55 (psa_utils.download): downloaded file Download-20210308184755.zip\n", "INFO 2021-03-08 18:47:55 (psa_utils.download): downloading product N20080804T023124677ID20F22.IMG\n", "INFO 2021-03-08 18:47:57 (psa_utils.download): downloaded file Download-20210308184755.zip\n" ] } ], "source": [ "download.download_by_lid('N20080804T023124677ID20F22', output_dir='/tmp')" ] }, { "cell_type": "markdown", "id": "excited-oracle", "metadata": {}, "source": [ "Note that all this really does is look for records with granule_uid matching the substring passed. So in this case the same data were archived as a PDS3 .IMG image and FITS file, so both were matched." ] }, { "cell_type": "markdown", "id": "foreign-maria", "metadata": {}, "source": [ "## download_labels_by_query\n", "\n", "Sometimes you cannot narrow down a search using TAP or PDAP, but need to look into the custom meta-data in the products. In this case you do not want to download gigabytes of data, but only the labels. \n", "\n", "This is a bit of a \"hack\" using the PDAP `files` endpoint with EPN-TAP, but it works with the following caveats:\n", "- large (especially PDS4) datasets currently time-out on PDAP - this is a known issue\n", "- PDS3 products with attached labels are skipped\n", " - since downloading these would defeat the purpose of trying to retrieve small labels only" ] }, { "cell_type": "code", "execution_count": 39, "id": "controlling-arlington", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO 2021-03-08 18:47:59 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170632_01_f__a0100.xml\n", "INFO 2021-03-08 18:47:59 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170738_02_f__t0005.xml\n", "INFO 2021-03-08 18:47:59 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170748_03_f__t0020.xml\n", "INFO 2021-03-08 18:47:59 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170758_04_f__t0040.xml\n", "INFO 2021-03-08 18:47:59 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170808_05_f__t0080.xml\n", "INFO 2021-03-08 18:48:00 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181020t170818_06_f__t0200.xml\n", "INFO 2021-03-08 18:48:00 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181026t103952_00_f__t0020.xml\n", "INFO 2021-03-08 18:48:00 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181027t060956_39_f__t0020.xml\n", "INFO 2021-03-08 18:48:01 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181027t061002_40_f__t0020.xml\n", "INFO 2021-03-08 18:48:01 (psa_utils.download): downloaded file cam_raw_sc_cam1_image_20181027t061008_41_f__t0020.xml\n" ] } ], "source": [ "download.download_labels_by_query(\"select top 10 * from epn_core where instrument_name='MCAM'\", output_dir='/tmp')" ] }, { "cell_type": "markdown", "id": "reserved-roman", "metadata": {}, "source": [ "# packager\n", "\n", "This is a module specifically designed to package PDS4 products into a delivery package recognised by the PSA. It is probably not useful unless you are an external data provider preparing PDS4 products (not bundles), or an internal user!" ] }, { "cell_type": "code", "execution_count": 40, "id": "frank-mounting", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mInit signature:\u001b[0m\n", "\u001b[0mpackager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPackager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'*.xml'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mrecursive\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mtemplate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0muse_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mclean\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m \n", "\u001b[0;31mInit docstring:\u001b[0m\n", "Initialise the packager class. Accepts the following:\n", "\n", "products - file pattern to match labels (*.xml default)\n", "input_dir - the root directory for the labels (default=.)\n", "use_dir - uses the product directory structure for the archive\n", " (if false, a minimal structure will be adopted)\n", "clean - if True, removes the generated files are generating the tarball\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/packager.py\n", "\u001b[0;31mType:\u001b[0m type\n", "\u001b[0;31mSubclasses:\u001b[0m \n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "packager.Packager?" ] }, { "cell_type": "markdown", "id": "sensitive-appendix", "metadata": {}, "source": [ "Most inputs should be clear - you need to specify the location and wildcard to match the input files - note that a delivery can only be for a single bundle, so use these filters to ensure you select products belonging to a single bundle!\n", "\n", "A template PDS4 label is needed for complete the product delivery label. A default template is needed, and bundled with the source code. But should you wish to specify a different template, this can be done with the template keyword.\n", "\n", "The main additional option is to set `use_dir`. If true, the path from `input_dir` will be used to build the archive structure (e.g. the directory structure users will see when downloading the files). If false, then all files will be placed in the root of the collection." ] }, { "cell_type": "markdown", "id": "stainless-welcome", "metadata": {}, "source": [ "# geogen\n", "\n", "This module contains some routines to help working with the PSA geometry generator (geogen). These are likely only of interest to internal users." ] }, { "cell_type": "markdown", "id": "infrared-spouse", "metadata": {}, "source": [ "## `generate_plf`" ] }, { "cell_type": "code", "execution_count": 41, "id": "false-collectible", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m\n", "\u001b[0mgeogen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_plf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mconfig_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mdirectory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mtable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mextras\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m\n", "Generates a GEOGEN plf input file.\n", "\n", "pds4_utils.Database() is used to scrape meta-data according to the config_file.\n", "files= specifies the label file pattern (defaults to *.xml)\n", "directory= specified the root of the input files (and the output location)\n", "table= specifies the table name in case the input file is configured to\n", " produce more than one. Default (None) assumes only one table.\n", "extras = a dictionary which provides extra static key/value pairs to be added\n", " to every entry (e.g. product type or similar). If an identical value exists\n", " in the table and extras, extras has priority.\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/geogen.py\n", "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "geogen.generate_plf?" ] }, { "cell_type": "markdown", "id": "embedded-drinking", "metadata": {}, "source": [ "Note that for this to work, `pds4_utils` must be installed. The `config_file` listed here is a `pds4_utils.Database` configuration file which is used to scrape relevant meta-data from a collection of PDS4 products and output to a json file in the format expected by geogen." ] }, { "cell_type": "markdown", "id": "raising-swaziland", "metadata": {}, "source": [ "# internal" ] }, { "cell_type": "markdown", "id": "willing-juvenile", "metadata": {}, "source": [ "## `Ingest_Test`\n", "\n", "This uses a configuration and template to effectively replicate a single PDS4 product according to the name, type, sub-instruments etc. in the configuration file" ] }, { "cell_type": "code", "execution_count": 42, "id": "cognitive-borough", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mInit signature:\u001b[0m\n", "\u001b[0minternal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIngest_Test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mconfig_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ingestion_test.yml'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mtemplate_label\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test_product.xml'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mpackage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m \n", "A class for generating test products from a label and data product\n", "template and a configuration file specifying the instrument-specific\n", "data\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/internal.py\n", "\u001b[0;31mType:\u001b[0m type\n", "\u001b[0;31mSubclasses:\u001b[0m \n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "internal.Ingest_Test?" ] }, { "cell_type": "markdown", "id": "equipped-genesis", "metadata": {}, "source": [ "## `build_context_json`\n", "\n", "This routine builds a local_context_products.json file as used by the PDS validate. It is designed for PSA internal use.\n", "\n", "It requires `pds4_utils`." ] }, { "cell_type": "code", "execution_count": 43, "id": "standard-coral", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m\n", "\u001b[0minternal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_context_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mconfig_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mjson_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'local_context_products.json'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mtable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'context_bundle'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m\n", "Generates a json file listing the name, type and LIDVID of all\n", "context files in input_dir. Generates a local context json file\n", "which can be used by the PDS validate tool and writes it to\n", "output_dir\n", "\n", "pds4_utils.Database() is used to scrape meta-data according to the config_file.\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/internal.py\n", "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "internal.build_context_json?" ] }, { "cell_type": "code", "execution_count": null, "id": "arbitrary-cement", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "responsible-scenario", "metadata": {}, "source": [ "## `collection_summary`\n", "\n", "This routine generates an html table corresponding to specific meta-data scraped from instrument and mission context products. It is used to prototype information which is needed o populate Google Dataset Search and for a DOI landing page." ] }, { "cell_type": "code", "execution_count": 44, "id": "modern-lunch", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\u001b[0;31mSignature:\u001b[0m\n", "\u001b[0minternal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollection_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mconfig_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mcontext_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m\n", "collection_summary accesses meta-data in a collection label\n", "or referenced from it, to produce a set of summary information\n", "needed to register a DOI and/or create a Google Dataset\n", "Search landing page.\n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/work/bepi/software/psa_utils/psa_utils/internal.py\n", "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "internal.collection_summary?" ] }, { "cell_type": "code", "execution_count": null, "id": "raised-dominican", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "bepi", "language": "python", "name": "bepi" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 5 }