{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "scrolled": false }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/nfs/slac/kipac/fs1/u/dmcat/software/anaconda2/envs/dmcat-gitdev/lib/python2.7/site-packages/matplotlib/__init__.py:1350: UserWarning: This call to matplotlib.use() has no effect\n", "because the backend has already been chosen;\n", "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n", "or matplotlib.backends is imported for the first time.\n", "\n", " warnings.warn(_use_error_msg)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Using /nfs/slac/kipac/fs1/u/dmcat/software/anaconda2/envs/dmcat-gitdev/lib/python2.7/site-packages/dmsky-0.2.1-py2.7.egg/dmsky/data/targets for TargetLibrary\n", "Using /gpfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/test_dmpipe/dmsky_data/targets for TargetLibrary\n" ] } ], "source": [ "from dmpipe.pipeline import Pipeline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "First we build a pipeline object to analyze the dSphs, and point it at the config file for the dSphs analysis." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "pipe = Pipeline(linkname = 'dSphs')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "configfile = 'config/master_dSphs.yaml'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We have to 'preconfigure' the pipeline, because we need to build up the list of targets so that we correctly set up the later stages of the pipeline." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "return code 0\n" ] } ], "source": [ "pipe.preconfigure(configfile)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We then tell the pipeline to update the arguments for all of the steps that comprise it." ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "scrolled": false }, "outputs": [], "source": [ "pipe.update_args(dict(config=configfile))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now we look around a bit, we can drill down into the links that make up the pipeline.\n", "In this case we have a pipeline that consists of a couple of preparation jobs,\n", "then 5 sub-pipelines ('data', 'sim_xxx', 'random').\n", "The sub-pipeline analyze the real data, and some different simulation cases.\n", "The 'random' sub-pipeline analyzes random sky directions." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "At this point the pipeline is fully configured, let's look around." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['prepare-targets',\n", " 'spec-table',\n", " 'data',\n", " 'sim_100GeV_bb_3e26',\n", " 'sim_null',\n", " 'sim_100GeV_bb_1e25',\n", " 'random',\n", " 'final-plots']" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe.linknames" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['analyze-roi',\n", " 'analyze-sed',\n", " 'convert-castro',\n", " 'stack-likelihood',\n", " 'plot-castro-sg',\n", " 'plot-dm-sg',\n", " 'plot-limits-sg',\n", " 'plot-stacked-dm-sg',\n", " 'plot-stacked-limits-sg']" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data'].linknames" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can print the status for the various links.\n", "Using recurse=True will also drill down to the status of the links in the sub pipelines." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Linkname : Link Status : Jobs Status\n", " prepare-targets : Done : Done\n", " spec-table : Ready : Pending\n", " data : Ready : Pending\n", " sim_100GeV_bb_3e26 : Ready : Pending\n", " sim_null : Ready : Pending\n", " sim_100GeV_bb_1e25 : Ready : Pending\n", " random : Ready : Pending\n", " final-plots : Ready : 8/ 0/ 0/ 0/ 0/ 8\n" ] } ], "source": [ "pipe.print_status()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Linkname : Link Status : Jobs Status\n", " prepare-targets : Done : Done\n", " spec-table : Ready : Pending\n", " data : Ready : Pending\n", "---------- data -----------\n", " Linkname : Link Status : Jobs Status\n", " analyze-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " analyze-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-castro-sg : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-dm-sg : Ready : 8/ 0/ 0/ 0/ 0/ 8\n", " plot-limits-sg : Ready : 8/ 0/ 0/ 0/ 0/ 8\n", " plot-stacked-dm-sg : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " plot-stacked-limits-sg : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " sim_100GeV_bb_3e26 : Ready : Pending\n", "---------- sim_100GeV_bb_3e26 -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " sim_null : Ready : Pending\n", "---------- sim_null -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " sim_100GeV_bb_1e25 : Ready : Pending\n", "---------- sim_100GeV_bb_1e25 -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " random : Ready : Pending\n", "---------- random -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " random-dir-gen : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " analyze-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", "------------------------------------------------\n", " final-plots : Ready : 8/ 0/ 0/ 0/ 0/ 8\n" ] } ], "source": [ "pipe.print_status(recurse=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can access a particular Link in the pipeline. \n", "We can ask:\n", " what the default options for the link are ('\\_options')\n", " what the current set of options are ('args')" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi']" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'action': ('run', 'Action to perform', str),\n", " 'check_status_once': (False,\n", " 'Check status only once before proceeding',\n", " bool),\n", " 'config': (None, 'Path to fermipy config file.', str),\n", " 'dry_run': (False, 'Print commands, but do not execute them', bool),\n", " 'job_check_sleep': (300,\n", " 'Sleep time between checking on job status (s)',\n", " int),\n", " 'make_plots': (False, 'Make plots', bool),\n", " 'print_update': (False, 'Print summary of job status', bool),\n", " 'roi_baseline': ('fit_baseline', 'Key for roi baseline file.', str),\n", " 'targetlist': (None, 'Path to the target list.', str),\n", " 'ttype': (None, 'Type of target being analyzed.', str)}" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi']._options" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'action': 'run',\n", " 'check_status_once': False,\n", " 'config': None,\n", " 'dry_run': False,\n", " 'job_check_sleep': 300,\n", " 'make_plots': False,\n", " 'print_update': False,\n", " 'roi_baseline': 'fit_baseline',\n", " 'targetlist': None,\n", " 'ttype': 'dSphs'}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].args" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can ask what jobs get run by this link. In this paritcular case the link\n", "runs 1 job that in turn dispatches several other jobs to the batch farm." ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "OrderedDict([('__top__@dSphs.data.analyze-roi',\n", " jobname : dSphs.data.analyze-roi\n", " jobkey : __top__\n", " appname : fermipy-analyze-roi-sg\n", " logfile : logs/dSphs.data.analyze-roi.log\n", " status : Ready)])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].jobs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we are talking to the link that represents any one of the disptached jobs.\n", "The command_template() function tells us how we would run this job from the UNIX command line." ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].scatter_link" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'fermipy-analyze-roi --roi_baseline {roi_baseline} --config {config}'" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].scatter_link.command_template()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here are ask what jobs will be dispatched. Note that there are two jobs with slightly different names." ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "OrderedDict([('draco@dSphs.data.analyze-roi',\n", " jobname : dSphs.data.analyze-roi\n", " jobkey : draco\n", " appname : fermipy-analyze-roi\n", " logfile : /nfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/dmpipe_example/dSphs/draco/analyze-roi_draco.log\n", " status : Ready),\n", " ('segue_1@dSphs.data.analyze-roi',\n", " jobname : dSphs.data.analyze-roi\n", " jobkey : segue_1\n", " appname : fermipy-analyze-roi\n", " logfile : /nfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/dmpipe_example/dSphs/segue_1/analyze-roi_segue_1.log\n", " status : Ready)])" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].scatter_link.jobs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here are are asking for information about the first of those two jobs. In particular, the specific options used for this instance of the job. You can merge the job_config with the command_template to get the exact syntax for the instance of this command." ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "jobname : dSphs.data.analyze-roi\n", " jobkey : draco\n", " appname : fermipy-analyze-roi\n", " logfile : /nfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/dmpipe_example/dSphs/draco/analyze-roi_draco.log\n", " status : Ready" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].scatter_link.jobs['draco@dSphs.data.analyze-roi']" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'config': './dSphs/draco/config.yaml',\n", " 'logfile': '/nfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/dmpipe_example/dSphs/draco/analyze-roi_draco.log',\n", " 'make_plots': False,\n", " 'roi_baseline': 'fit_baseline'}" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['analyze-roi'].scatter_link.jobs['draco@dSphs.data.analyze-roi'].job_config" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we run a single link. \n", "We use 'run_with_log()' so that the log file will be in the correct place and the pipeline will know the job\n", "has completed. " ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "return code 0\n" ] } ], "source": [ "pipe['spec-table'].run_with_log()" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Linkname : Link Status : Jobs Status\n", " prepare-targets : Done : Done\n", " spec-table : Done : Done\n", " data : Ready : Pending\n", " sim_100GeV_bb_3e26 : Ready : Pending\n", " sim_null : Ready : Pending\n", " sim_100GeV_bb_1e25 : Ready : Pending\n", " random : Ready : Pending\n", " final-plots : Ready : 8/ 0/ 0/ 0/ 0/ 8\n" ] } ], "source": [ "pipe.print_status()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we run the 'data' sub-pipeline." ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running link dSphs.data.analyze-roi\n", "Submitting jobs (2): !\n", "Checking status (300s): ...! Done\n", "Running link dSphs.data.analyze-sed\n", "Submitting jobs (2): !\n", "Checking status (300s): .! Done\n", "Running link dSphs.data.convert-castro\n", "Submitting jobs (4): !\n", "Checking status (120s): .! Done\n", "Running link dSphs.data.stack-likelihood\n", "Submitting jobs (2): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.data.plot-castro-sg\n", "Submitting jobs (2): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.data.plot-dm-sg\n", "Submitting jobs (8): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.data.plot-limits-sg\n", "Submitting jobs (8): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.data.plot-stacked-dm-sg\n", "Submitting jobs (4): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.data.plot-stacked-limits-sg\n", "Submitting jobs (4): !\n", "Checking status (60s): .! Done\n", "Chain status 5\n", "Timestamp: 1527130057\n", "Successfully completed\n" ] } ], "source": [ "pipe['data'].run(resubmit_failed=True)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Linkname : Link Status : Jobs Status\n", " prepare-targets : Done : Done\n", " spec-table : Done : Done\n", " data : Done : Done\n", "---------- data -----------\n", " Linkname : Link Status : Jobs Status\n", " analyze-roi : Done : 0/ 0/ 0/ 2/ 0/ 2\n", " analyze-sed : Done : 0/ 0/ 0/ 2/ 0/ 2\n", " convert-castro : Done : 0/ 0/ 0/ 4/ 0/ 4\n", " stack-likelihood : Done : 0/ 0/ 0/ 2/ 0/ 2\n", " plot-castro-sg : Done : 0/ 0/ 0/ 2/ 0/ 2\n", " plot-dm-sg : Done : 0/ 0/ 0/ 8/ 0/ 8\n", " plot-limits-sg : Done : 0/ 0/ 0/ 8/ 0/ 8\n", " plot-stacked-dm-sg : Done : 0/ 0/ 0/ 4/ 0/ 4\n", " plot-stacked-limits-sg : Done : 0/ 0/ 0/ 4/ 0/ 4\n", "------------------------------------------------\n", " sim_100GeV_bb_3e26 : Ready : Pending\n", "---------- sim_100GeV_bb_3e26 -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " sim_null : Ready : Pending\n", "---------- sim_null -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " sim_100GeV_bb_1e25 : Ready : Pending\n", "---------- sim_100GeV_bb_1e25 -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " simulate-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-control-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", "------------------------------------------------\n", " random : Ready : Pending\n", "---------- random -----------\n", " Linkname : Link Status : Jobs Status\n", " copy-base-roi : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " random-dir-gen : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " analyze-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " convert-castro : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " stack-likelihood : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-sed : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " collect-limits : Ready : 4/ 0/ 0/ 0/ 0/ 4\n", " collect-stacked-limits : Ready : 2/ 0/ 0/ 0/ 0/ 2\n", " plot-stacked-dm : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", " plot-stacked-limits : Ready : 80/ 0/ 0/ 0/ 0/ 80\n", "------------------------------------------------\n", " final-plots : Ready : 8/ 0/ 0/ 0/ 0/ 8\n" ] } ], "source": [ "pipe.print_status(recurse=True)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'clobber': False,\n", " 'j_value_file': './dSphs/draco/j_val_ack2016.yaml',\n", " 'jprior': 'lgauss',\n", " 'limitfile': './dSphs/draco/dmlimits_ack2016_point_lgauss.fits',\n", " 'logfile': '/nfs/slac/kipac/fs1/u/dmcat/workdir/sandbox/dmpipe_example/dSphs/draco/dmlike_ack2016_point_lgauss.log',\n", " 'nsims': -1,\n", " 'outfile': './dSphs/draco/dmlike_ack2016_point_lgauss.fits',\n", " 'sed_file': './dSphs/draco/sed_ack2016_point.fits',\n", " 'seed': -1,\n", " 'specfile': './dm_spectra_dSphs.fits'}" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pipe['data']['convert-castro'].scatter_link.jobs['draco:ack2016_point:lgauss@dSphs.data.convert-castro'].job_config" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Skipping done link dSphs.prepare-targets\n", "Skipping done link dSphs.spec-table\n", "Running link dSphs.data.data\n", "Skipping done link dSphs.data.analyze-roi\n", "Skipping done link dSphs.data.analyze-sed\n", "Skipping done link dSphs.data.convert-castro\n", "Skipping done link dSphs.data.stack-likelihood\n", "Skipping done link dSphs.data.plot-castro-sg\n", "Skipping done link dSphs.data.plot-dm-sg\n", "Skipping done link dSphs.data.plot-limits-sg\n", "Skipping done link dSphs.data.plot-stacked-dm-sg\n", "Skipping done link dSphs.data.plot-stacked-limits-sg\n", "Chain status 5\n", "Running link dSphs.sim_100GeV_bb_3e26.sim_100GeV_bb_3e26\n", "Running link dSphs.sim_100GeV_bb_3e26.copy-base-roi\n", "Submitting jobs (2): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.sim_100GeV_bb_3e26.simulate-roi\n", "Submitting jobs (2): !\n", "Checking status (300s): ......! Done\n", "Running link dSphs.sim_100GeV_bb_3e26.convert-castro\n", "Submitting jobs (4): !\n", "Checking status (120s): ...! Done\n", "Running link dSphs.sim_100GeV_bb_3e26.stack-likelihood\n", "Submitting jobs (2): !\n", "Checking status (60s): ..! Done\n", "Running link dSphs.sim_100GeV_bb_3e26.collect-sed\n", "Submitting jobs (2): !\n", "Checking status (60s): .! Done\n", "Running link dSphs.sim_100GeV_bb_3e26.collect-limits\n", "Submitting jobs (4): !\n", "Checking status (60s): ...! Partially Failed\n", "Stoping chain execution at failed link dSphs.sim_100GeV_bb_3e26.collect-limits\n", "Chain status 7\n", "Stoping chain execution at failed link dSphs.sim_100GeV_bb_3e26.sim_100GeV_bb_3e26\n", "Chain status 7\n", "Timestamp: 1527132646\n", "Exited with exit code -1\n" ] } ], "source": [ "pipe.run(resubmit_failed=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 2 }