Figure 1:
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Figure 2 is slightly different: we need to enable gradient colors under options, and change the uninformative ages (None/NA/0.0) to a grey." ] }, { "cell_type": "code", "collapsed": false, "input": [ "FileLink(fig2, result_html_prefix='Figure 2:
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "FileLink(fig3, result_html_prefix='Figure 3:
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Once the images are created, we need to copy them to our working area. It is possible file paths will be different, in which case, you may need to change the DOWNLOAD_DIRECTORY variable in the next cell." ] }, { "cell_type": "code", "collapsed": false, "input": [ "DOWNLOAD_DIRECTORY = os.path.expandvars('$HOME')\n", "FIGURE1_EXPECTED_FILENAME = 'Figure_1.tar.gz'\n", "FIGURE2_EXPECTED_FILENAME = 'Figure_2.tar.gz'\n", "FIGURE3_EXPECTED_FILENAME = 'Figure_3.tar.gz'\n", "\n", "# helper function for creating wildcard paths\n", "source_path = lambda x,y: os.path.join(x,y)\n", "\n", "# setup the destination paths\n", "emperor_images = get_relative_new_path('emperor_images_svg')\n", "\n", "if not os.path.exists(emperor_images):\n", " os.mkdir(emperor_images)\n", "\n", "# setup the source paths\n", "figure1_src = source_path(DOWNLOAD_DIRECTORY, FIGURE1_EXPECTED_FILENAME)\n", "figure2_src = source_path(DOWNLOAD_DIRECTORY, FIGURE2_EXPECTED_FILENAME)\n", "figure3_src = source_path(DOWNLOAD_DIRECTORY, FIGURE3_EXPECTED_FILENAME)\n", "\n", "check_file(figure1_src)\n", "check_file(figure2_src)\n", "check_file(figure3_src)\n", "\n", "# unpack the tarballs\n", "jobs = []\n", "jobs.append(submit('tar xzf %s -C %s' % (figure1_src, emperor_images)))\n", "jobs.append(submit('tar xzf %s -C %s' % (figure2_src, emperor_images)))\n", "jobs.append(submit('tar xzf %s -C %s' % (figure3_src, emperor_images)))\n", "res = wait_on(jobs)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now we need to produce the actual PDF files of the PCoA plots that will go into the individualized results." ] }, { "cell_type": "code", "collapsed": false, "input": [ "all_ag_IDs = set([l.strip().split('\\t')[0] for l in open(ag_100nt_m_fp) if not l.startswith('#')])\n", "all_emp_SVGs = os.listdir(emperor_images)\n", "template_files = get_relative_new_path('template_files')\n", "\n", "if not os.path.exists(template_files):\n", " os.mkdir(template_files)\n", "\n", "svg_smash_args = {'input':emperor_images, \n", " 'output':template_files, \n", " 'prefix':None, \n", " 'sample_id':None}\n", "\n", "commands = construct_svg_smash_commands(all_emp_SVGs, all_ag_IDs, scripts['SVG Smash'], svg_smash_args)\n", "res = farm_commands(commands, 25)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now lets create the phylum level taxonomy summary plots." ] }, { "cell_type": "code", "collapsed": false, "input": [ "fig4_sex_fp = get_relative_new_path('fig4_sex.biom')\n", "fig4_age_fp = get_relative_new_path('fig4_age_category.biom')\n", "fig4_diet_fp = get_relative_new_path('fig4_diet.biom')\n", "fig4_bmi_fp = get_relative_new_path('fig4_bmi_category.biom')\n", "ag_fecal_t_1k_fp = get_relative_new_path('ag_fecal_even1k.biom')\n", "ag_oral_t_1k_fp = get_relative_new_path('ag_oral_even1k.biom')\n", "ag_skin_t_1k_fp = get_relative_new_path('ag_skin_even1k.biom')\n", "template_files = get_relative_existing_path('template_files')\n", "\n", "check_file(hmp_gg_ag_pgp_t_1k_fp)\n", "check_file(fig4_m_fp)\n", "check_file(oral_m_fp)\n", "check_file(skin_m_fp)\n", "\n", "filter_fecal_args = {'input':hmp_gg_ag_pgp_t_1k_fp,\n", " 'output':ag_fecal_t_1k_fp,\n", " 'sample_id_fp':fig4_m_fp}\n", "\n", "filter_oral_args = {'input':hmp_gg_ag_pgp_t_1k_fp,\n", " 'output':ag_oral_t_1k_fp,\n", " 'sample_id_fp':oral_m_fp}\n", "\n", "filter_skin_args = {'input':hmp_gg_ag_pgp_t_1k_fp,\n", " 'output':ag_skin_t_1k_fp,\n", " 'sample_id_fp':skin_m_fp}\n", "\n", "otu_by_cat_sex_args = {'otu_table':ag_fecal_t_1k_fp,\n", " 'output':fig4_sex_fp,\n", " 'mapping':fig4_m_fp,\n", " 'category':'SEX'}\n", "otu_by_cat_age_args = {'otu_table':ag_fecal_t_1k_fp,\n", " 'output':fig4_age_fp,\n", " 'mapping':fig4_m_fp,\n", " 'category':'AGE_CATEGORY'}\n", "otu_by_cat_diet_args = {'otu_table':ag_fecal_t_1k_fp,\n", " 'output':fig4_diet_fp,\n", " 'mapping':fig4_m_fp,\n", " 'category':'DIET_TYPE'}\n", "otu_by_cat_bmi_args = {'otu_table':ag_fecal_t_1k_fp,\n", " 'output':fig4_bmi_fp,\n", " 'mapping':fig4_m_fp,\n", " 'category':'BMI_CATEGORY'}\n", "\n", "phyla_plot_args = {'input':ag_fecal_t_1k_fp,\n", " 'output':template_files,\n", " 'mapping':fig4_m_fp,\n", " 'debug':\"-d\" if debug else \"\",\n", " 'categories':'SEX:%s, AGE_CATEGORY:%s, DIET_TYPE:%s, BMI_CATEGORY:%s' % (fig4_sex_fp, fig4_age_fp, fig4_diet_fp, fig4_bmi_fp)}\n", " \n", "# filter the ag table down to just the fecal samples for fig 4\n", "filter_jobs = []\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_fecal_args))\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_oral_args))\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_skin_args))\n", "res = wait_on(filter_jobs)\n", "\n", "# get the summarized tables\n", "jobs = []\n", "jobs.append(submit(scripts['Summarize OTU by Category'] % otu_by_cat_sex_args))\n", "jobs.append(submit(scripts['Summarize OTU by Category'] % otu_by_cat_age_args))\n", "jobs.append(submit(scripts['Summarize OTU by Category'] % otu_by_cat_diet_args))\n", "jobs.append(submit(scripts['Summarize OTU by Category'] % otu_by_cat_bmi_args))\n", "res = wait_on(jobs)\n", "\n", "# farm out the phyla plots\n", "sample_ids = [l.split('\\t')[0] for l in open(fig4_m_fp) if not l.startswith('#')]\n", "commands = construct_phyla_plots_cmds(sample_ids, scripts['Make Phyla Plots'], phyla_plot_args)\n", "res = farm_commands(commands, 5)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And for the last figure, lets create the over represented taxonomy table." ] }, { "cell_type": "code", "collapsed": false, "input": [ "ag_fecal_t_norare_fp = get_relative_new_path('ag_fecal_norare_filtered.biom')\n", "ag_skin_t_norare_fp = get_relative_new_path('ag_skin_norare_filtered.biom')\n", "ag_oral_t_norare_fp = get_relative_new_path('ag_oral_norare_filtered.biom')\n", "\n", "fecal_tax_sum = get_relative_new_path('ag_fecal_norare_taxasum')\n", "skin_tax_sum = get_relative_new_path('ag_skin_norare_taxasum')\n", "oral_tax_sum = get_relative_new_path('ag_oral_norare_taxasum')\n", "\n", "template_files = get_relative_existing_path('template_files')\n", "\n", "filter_fecal_args = {'input':ag_100nt_t_fp,\n", " 'output':ag_fecal_t_norare_fp,\n", " 'sample_id_fp':fig4_m_fp}\n", "\n", "filter_skin_args = {'input':ag_100nt_t_fp,\n", " 'output':ag_skin_t_norare_fp,\n", " 'sample_id_fp':skin_m_fp}\n", "\n", "filter_oral_args = {'input':ag_100nt_t_fp,\n", " 'output':ag_oral_t_norare_fp,\n", " 'sample_id_fp':oral_m_fp}\n", "\n", "sum_fecal_args = {'input':ag_fecal_t_norare_fp,\n", " 'output':fecal_tax_sum,\n", " 'level':\"2,3,6\"}\n", "\n", "sum_skin_args = {'input':ag_skin_t_norare_fp,\n", " 'output':skin_tax_sum,\n", " 'level':\"2,3,6\"}\n", "\n", "sum_oral_args = {'input':ag_oral_t_norare_fp,\n", " 'output':oral_tax_sum,\n", " 'level':\"2,3,6\"}\n", "\n", "filter_jobs = []\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_fecal_args))\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_skin_args))\n", "filter_jobs.append(submit(scripts['Filter Samples'] % filter_oral_args))\n", "res = wait_on(filter_jobs)\n", "\n", "sum_jobs = []\n", "sum_jobs.append(submit(scripts['Summarize Taxa'] % sum_fecal_args))\n", "sum_jobs.append(submit(scripts['Summarize Taxa'] % sum_skin_args))\n", "sum_jobs.append(submit(scripts['Summarize Taxa'] % sum_oral_args))\n", "res = wait_on(sum_jobs)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "sum_fecal_args = {'input':ag_fecal_t_norare_fp,\n", " 'output':fecal_tax_sum,\n", " 'level':\"2,3,6\"}\n", "res = wait_on(submit(scripts['Summarize Taxa'] % sum_fecal_args))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "template_files = get_relative_existing_path('template_files')\n", "fig5_lvl6 = get_relative_existing_path('ag_fecal_norare_taxasum/ag_fecal_norare_filtered_L6.biom')\n", "\n", "sig_args = {'input':fig5_lvl6,\n", " 'output':template_files}\n", "\n", "from time import sleep\n", "jobs = []\n", "sample_ids = [l.split('\\t')[0] for l in open(fig4_m_fp) if not l.startswith('#')]\n", "for id_ in sample_ids:\n", " args = sig_args.copy()\n", " args['samples'] = id_\n", " cmd = scripts['OTU Significance'] % args\n", " jobs.append(submit(cmd))\n", " sleep(0.1)\n", "res = wait_on(jobs)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Lets also dump out the per sample taxonomy information that we can distribute on the participant website." ] }, { "cell_type": "code", "collapsed": false, "input": [ "from biom.util import biom_open\n", "with biom_open(fig5_lvl6) as table:\n", " per_sample_taxa_summaries(table, get_relative_new_path('template_files/Figure_6_%s.txt'))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we construct a method that will return the necessary commands to populate a template per sample." ] }, { "cell_type": "code", "collapsed": false, "input": [ "static_paths = {'template': template,\n", " 'aglogo':aglogo,\n", " 'fig1_legend': fig1_legend,\n", " 'fig2_legend': fig2_legend,\n", " 'fig2_2ndlegend': fig2_2ndlegend,\n", " 'fig3_legend': fig3_legend,\n", " 'fig4_overlay': fig4_overlay,\n", " 'fig1_ovals': fig1_ovals,\n", " 'fig2_ovals': fig2_ovals,\n", " 'ball_legend': ball_legend,\n", " 'title': title,\n", " 'working_dir': working_dir}" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now lets build up the list of commands that will populate the templates per sample." ] }, { "cell_type": "code", "collapsed": false, "input": [ "check_file(ag_100nt_m_massaged_fp)\n", "check_file(template)\n", "\n", "if not os.path.exists(get_relative_new_path('unidentified')):\n", " os.mkdir(get_relative_new_path('unidentified'))\n", "if participants is not None and os.path.exists(get_relative_new_path('identified')):\n", " os.mkdir(get_relative_new_path('identified'))\n", " \n", "base_setup_cmd = 'cd %s; %s'\n", "\n", "indiv_cmds, latex_cmds, missing = construct_bootstrap_and_latex_commands(all_ag_IDs, participants, \n", " get_relative_existing_path,\n", " static_paths, base_setup_cmd, \n", " scripts['To PDF'])\n", "_ = farm_commands(indiv_cmds, 25)\n", "_ = farm_commands(latex_cmds, 25)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "These last few cells \"harvest\" the populated templates, which places them in a single folder. We can then \"smash\" or combine multiple templates together to simplify the results printing process." ] }, { "cell_type": "code", "collapsed": false, "input": [ "if participants is not None:\n", " res = farm_commands(harvest(get_relative_existing_path('identified')), 50)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "res = farm_commands(harvest(get_relative_existing_path('unidentified')), 50)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We only do this part if we have identifying information on hand." ] }, { "cell_type": "code", "collapsed": false, "input": [ "if participants is not None:\n", " harvest_path = get_path('identified/harvested')\n", " identified_smash = pdf_smash(harvest_path, 'identified', previously_printed=prev_printed)\n", " res = farm_commands(identified_smash, 1)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we're done! You can view all of the intermediate files in the working directory which will be displayed below." ] }, { "cell_type": "code", "collapsed": false, "input": [ "print working_dir" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next up, we're going to create the project summary information." ] }, { "cell_type": "code", "collapsed": false, "input": [ "# setup paths\n", "key_taxa = get_relative_existing_path('stacked_plots_key_taxa.txt')\n", "\n", "fecal_tax_sum_1k = get_relative_new_path('ag_fecal_1k_taxasum')\n", "oral_tax_sum_1k = get_relative_new_path('ag_oral_1k_taxasum')\n", "skin_tax_sum_1k = get_relative_new_path('ag_skin_1k_taxasum')\n", "\n", "fecal_tax_sum_1k_phy = get_relative_new_path('ag_fecal_1k_taxasum/ag_fecal_even1k_L2.txt')\n", "oral_tax_sum_1k_phy = get_relative_new_path('ag_oral_1k_taxasum/ag_oral_even1k_L2.txt')\n", "skin_tax_sum_1k_phy = get_relative_new_path('ag_skin_1k_taxasum/ag_skin_even1k_L2.txt')\n", "\n", "uw_unif_dm_fecal = get_relative_new_path('unweighted_unifrac_HMP_GG_AG_PGP_100nt_even1k_fecal.txt')\n", "uw_unif_dm_fecal_hmp = get_relative_new_path('unweighted_unifrac_HMP_even1k_fecal.txt')\n", "uw_unif_dm_fecal_agp = get_relative_new_path('unweighted_unifrac_AGP_even1k_fecal.txt')\n", "w_unif_dm_fecal = get_relative_new_path('weighted_unifrac_HMP_GG_AG_PGP_100nt_even1k_fecal.txt')\n", "w_unif_dm_fecal_hmp = get_relative_new_path('weighted_unifrac_HMP_even1k_fecal.txt')\n", "w_unif_dm_fecal_agp = get_relative_new_path('weighted_unifrac_AGP_even1k_fecal.txt')\n", "\n", "qiime_charts_config = get_relative_existing_path('metadata_charts.json')\n", "\n", "fig5 = get_relative_new_path('fig5.pdf')\n", "\n", "jobs = []\n", "\n", "# get summary counts\n", "agp_sample_count = count_samples(open(mapping_fp))\n", "agp_seq_count = count_seqs(open(merged_sequences_full_length_fp))\n", "agp_unique_participants = count_unique_participants(open(mapping_fp))\n", "pgp_sample_count = count_samples(open(ag_100nt_m_massaged_fp), criteria={'IS_PGP': 'Yes'})\n", "pgp_seq_count = count_seqs(open(bloom_filtered_100nt), subset=pgp_ids)\n", "pgp_unique_participants = count_unique_participants(open(ag_100nt_m_massaged_fp), criteria={'IS_PGP': 'Yes'})\n", "\n", "# from the GET2012 sampling which were not part of the American Gut\n", "pgp_sample_count += 439\n", "pgp_unique_participants += 86\n", "pgp_seq_count += 9509776\n", "\n", "# summarize taxa for the stacked taxa plots\n", "sum_fecal_args = {'input':ag_fecal_t_1k_fp,\n", " 'output':fecal_tax_sum_1k,\n", " 'level':\"2\"}\n", "\n", "sum_oral_args = {'input':ag_oral_t_1k_fp,\n", " 'output':oral_tax_sum_1k,\n", " 'level':\"2\"}\n", "\n", "sum_skin_args = {'input':ag_skin_t_1k_fp,\n", " 'output':skin_tax_sum_1k,\n", " 'level':\"2\"}\n", "jobs.append(submit(scripts['Summarize Taxa'] % sum_fecal_args))\n", "jobs.append(submit(scripts['Summarize Taxa'] % sum_oral_args))\n", "jobs.append(submit(scripts['Summarize Taxa'] % sum_skin_args))\n", "res = wait_on(jobs)\n", "\n", "jobs = []\n", "\n", "# setup the stacked taxa plots commands\n", "agplots_args = {'mapping': ag_100nt_m_massaged_fp,\n", " 'taxa': None,\n", " 'metadata_cat': 'SIMPLE_BODY_SITE',\n", " 'metadata_val': None,\n", " 'output_prefix': None,\n", " 'key_taxa': key_taxa}\n", "\n", "agplots_fecal = agplots_args.copy()\n", "agplots_fecal['taxa'] = fecal_tax_sum_1k_phy\n", "agplots_fecal['metadata_val'] = 'FECAL'\n", "agplots_fecal['output_prefix'] = 'ag_plots_fecal_'\n", "agplots_fecal['identified'] = 'fecal_identified.txt'\n", "\n", "agplots_oral = agplots_args.copy()\n", "agplots_oral['taxa'] = oral_tax_sum_1k_phy\n", "agplots_oral['metadata_val'] = 'ORAL'\n", "agplots_oral['output_prefix'] = 'ag_plots_oral_'\n", "agplots_oral['identified'] = 'oral_identified.txt'\n", "\n", "agplots_skin = agplots_args.copy()\n", "agplots_skin['taxa'] = skin_tax_sum_1k_phy\n", "agplots_skin['metadata_val'] = 'SKIN'\n", "agplots_skin['output_prefix'] = 'ag_plots_skin_'\n", "agplots_skin['identified'] = 'skin_identified.txt'\n", "\n", "# setup the initial filtering for the beta diversity HMP/AGP comparison\n", "filter_uwdm_to_fecal = {'input': fig1_bdiv,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'SIMPLE_BODY_SITE:FECAL',\n", " 'output': uw_unif_dm_fecal}\n", "filter_wdm_to_fecal = {'input': w_bdiv_path,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'SIMPLE_BODY_SITE:FECAL',\n", " 'output': w_unif_dm_fecal}\n", "\n", "# submit the stacked taxa plots, metadata summaries (QIIME Charts) and the initial beta diversity filtering commands\n", "jobs.append(submit(scripts['AGPlots'] % agplots_fecal))\n", "jobs.append(submit(scripts['AGPlots'] % agplots_oral))\n", "jobs.append(submit(scripts['AGPlots'] % agplots_skin))\n", "jobs.append(submit(scripts['QIIME Charts'] % {'input': qiime_charts_config}))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_fecal))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_wdm_to_fecal))\n", "\n", "res = wait_on(jobs)\n", "\n", "jobs = []\n", "\n", "# second round of beta diversity filtering into project specific results\n", "filter_uwdm_to_fecal_hmp = {'input': uw_unif_dm_fecal,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:HMP',\n", " 'output': uw_unif_dm_fecal_hmp}\n", "\n", "filter_uwdm_to_fecal_agp = {'input': uw_unif_dm_fecal,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:AGP',\n", " 'output': uw_unif_dm_fecal_agp}\n", "\n", "filter_wdm_to_fecal_hmp = {'input': w_unif_dm_fecal,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:HMP',\n", " 'output': w_unif_dm_fecal_hmp}\n", "\n", "filter_wdm_to_fecal_agp = {'input': w_unif_dm_fecal,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:AGP',\n", " 'output': w_unif_dm_fecal_agp}\n", "\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_fecal_hmp))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_fecal_agp))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_wdm_to_fecal_hmp))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_wdm_to_fecal_agp))\n", "res = wait_on(jobs)\n", "\n", "# perform the beta diversity comparison of the HMP and AGP\n", "bdiv_compare_w = {'inputs': ','.join([w_unif_dm_fecal_hmp, w_unif_dm_fecal_agp]),\n", " 'output': fig5 + '.dtest.pdf',\n", " 'labels': 'HMP,AGP',\n", " 'title': \"'Beta diversity added by sampled microbial communities'\",\n", " 'ylabel': \"'Diversity (weighted UniFrac)'\"}\n", "res = wait_on(submit(scripts['Beta Diversity Comparison'] % bdiv_compare_w))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next, we're going to get the PCoA shots sorted out. For this, we need a few different slices of beta diversity. First, we want to compare the AGP against the HMP, the AGP against the PGP, and finally the US fecal samples of the AGP against the Global Gut. We already have that last one computed though. " ] }, { "cell_type": "code", "collapsed": false, "input": [ "pc_path = lambda x: x.rsplit('.txt',1)[0] + '_pc.txt'\n", "emp_path = lambda x: x.rsplit('.txt',1)[0] + '-emp'\n", "emp_index = lambda x: os.path.join(x, 'index.html')\n", "\n", "# filter beta diversity to just HMP and AGP\n", "uw_unif_dm_hmp_and_agp = get_relative_new_path('unweighted_unifrac_HMP_AG_100nt_even1k.txt')\n", "uw_unif_dm_pgp_and_agp = get_relative_new_path('unweighted_unifrac_PGP_AG_100nt_even1k.txt')\n", "uw_unif_dm_gg_and_agp = get_relative_new_path('unweighted_unifrac_GG_AG_100nt_even1k.txt')\n", "\n", "filter_uwdm_to_hmp_and_agp = {'input': fig1_bdiv,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:HMP,AGP',\n", " 'output': uw_unif_dm_hmp_and_agp}\n", "\n", "filter_uwdm_to_pgp_and_agp = {'input': fig1_bdiv,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': 'TITLE_ACRONYM:AGP,PGP',\n", " 'output': uw_unif_dm_pgp_and_agp}\n", "# just show US/Venezuela/Malawi\n", "filter_uwdm_to_gg_and_agp = {'input': fig2_bdiv,\n", " 'mapping': hmp_pgp_ag_gg_mm_fp,\n", " 'states': \"'COUNTRY:United States of America,Malawi,Venezuela'\",\n", " 'output': uw_unif_dm_gg_and_agp}\n", "\n", "jobs = []\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_hmp_and_agp))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_pgp_and_agp))\n", "jobs.append(submit(scripts['Filter Distance Matrix by Metadata'] % filter_uwdm_to_gg_and_agp))\n", "res = wait_on(jobs)\n", "\n", "# compute principal coordinates \n", "uw_unif_pc_hmp_and_agp = pc_path(uw_unif_dm_hmp_and_agp)\n", "uw_unif_pc_pgp_and_agp = pc_path(uw_unif_dm_pgp_and_agp)\n", "uw_unif_pc_gg_and_agp = pc_path(uw_unif_dm_gg_and_agp)\n", "\n", "hmp_agp_pc_cmd_args = {'input':uw_unif_dm_hmp_and_agp,\n", " 'output':uw_unif_pc_hmp_and_agp}\n", "pgp_agp_pc_cmd_args = {'input':uw_unif_dm_pgp_and_agp,\n", " 'output':uw_unif_pc_pgp_and_agp}\n", "gg_agp_pc_cmd_args = {'input':uw_unif_dm_gg_and_agp,\n", " 'output':uw_unif_pc_gg_and_agp}\n", "\n", "jobs = []\n", "jobs.append(submit(scripts['Principal Coordinates'] % hmp_agp_pc_cmd_args))\n", "jobs.append(submit(scripts['Principal Coordinates'] % pgp_agp_pc_cmd_args))\n", "jobs.append(submit(scripts['Principal Coordinates'] % gg_agp_pc_cmd_args))\n", "res = wait_on(jobs)\n", "\n", "# produce PCoAs\n", "uw_unif_emp_hmp_and_agp = emp_path(uw_unif_pc_hmp_and_agp)\n", "uw_unif_emp_pgp_and_agp = emp_path(uw_unif_pc_pgp_and_agp)\n", "uw_unif_emp_gg_and_agp = emp_path(uw_unif_pc_gg_and_agp)\n", "\n", "hmp_agp_emp_cmd_args = {'input':uw_unif_pc_hmp_and_agp, \n", " 'output':uw_unif_emp_hmp_and_agp, \n", " 'mapping':fig1_m_fp}\n", "pgp_agp_emp_cmd_args = {'input':uw_unif_pc_pgp_and_agp, \n", " 'output':uw_unif_emp_pgp_and_agp, \n", " 'mapping':fig1_m_fp}\n", "gg_agp_emp_cmd_args = {'input':uw_unif_pc_gg_and_agp, \n", " 'output':uw_unif_emp_gg_and_agp, \n", " 'mapping':fig2_m_fp}\n", "\n", "jobs = []\n", "jobs.append(submit(scripts['Make Emperor'] % hmp_agp_emp_cmd_args))\n", "jobs.append(submit(scripts['Make Emperor'] % pgp_agp_emp_cmd_args))\n", "jobs.append(submit(scripts['Make Emperor'] % gg_agp_emp_cmd_args))\n", "res = wait_on(jobs)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As before with the PCoAs for the individual plots, we'll need to save SVGs and transfer them back. Similar to how we did the PCoAs for the participant results, we'll need to do some manual work here.\n", "\n", "* For the first figure, set TITLE_BODY_SITE, color and save the resulting figure as a SVG named \"agp_hmp\"\n", "* For the second figure, set TITLE_BODY_SITE, color and save the resulting figure as a SVG named \"agp_pgp\"\n", "* For the third figures, first set AGE, color as a gradient, remove points with values \"no_data\", \"None\", \"0.0\", and \"na\", then save a SVG as \"agp_gg_age\". Next, set COUNTRY and save as \"agp_gg_country\"\n", "\n", "The current colors being used are:\n", "\n", "* AGP-Fecal - 0x9c0b18\n", "* AGP-Oral - 0x23529a\n", "* AGP-Skin - 0xee8735\n", "* HMP/PGP-Fecal - 0xe75a44\n", "* HMP/PGP-Oral - 0x74c7df\n", "* HMP/PGP-Skin - 0xf3f5a0\n", "\n", "Once these images are saved, copy them back to $HOME." ] }, { "cell_type": "code", "collapsed": false, "input": [ "FileLink(emp_index(uw_unif_emp_hmp_and_agp), result_html_prefix='HMP and AGP, please save an SVG as \"agp_hmp\"
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "FileLink(emp_index(uw_unif_emp_pgp_and_agp), result_html_prefix='PGP and AGP, please save an SVG as \"agp_pgp\"
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "code", "collapsed": false, "input": [ "FileLink(emp_index(uw_unif_emp_gg_and_agp), result_html_prefix='GG and AGP, please save an SVG as \"agp_gg_age\" and \"agp_gg_country\"
')" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now lets convert the SVGs to PDFs which greatly reduces the size of the resulting populated template." ] }, { "cell_type": "code", "collapsed": false, "input": [ "# SVG to PDF\n", "agp_hmp_svg = source_path(DOWNLOAD_DIRECTORY, 'agp_hmp.svg')\n", "agp_pgp_svg = source_path(DOWNLOAD_DIRECTORY, 'agp_pgp.svg')\n", "agp_gg_age_svg = source_path(DOWNLOAD_DIRECTORY, 'agp_gg_age.svg')\n", "agp_gg_country_svg = source_path(DOWNLOAD_DIRECTORY, 'agp_gg_country.svg')\n", "\n", "check_file(agp_hmp_svg)\n", "check_file(agp_pgp_svg)\n", "check_file(agp_gg_age_svg)\n", "check_file(agp_gg_country_svg)\n", "\n", "agp_hmp_pdf = get_relative_new_path('agp_hmp.pdf')\n", "agp_pgp_pdf = get_relative_new_path('agp_pgp.pdf')\n", "agp_gg_age_pdf = get_relative_new_path('agp_gg_age.pdf')\n", "agp_gg_country_pdf = get_relative_new_path('agp_gg_country.pdf')\n", "\n", "agp_hmp_args = {'input': agp_hmp_svg, 'output': agp_hmp_pdf}\n", "agp_pgp_args = {'input': agp_pgp_svg, 'output': agp_pgp_pdf}\n", "agp_gg_age_args = {'input': agp_gg_age_svg, 'output': agp_gg_age_pdf}\n", "agp_gg_country_args = {'input': agp_gg_country_svg, 'output': agp_gg_country_pdf}\n", "\n", "jobs = []\n", "jobs.append(submit(scripts['SVG to PDF'] % agp_hmp_args))\n", "jobs.append(submit(scripts['SVG to PDF'] % agp_pgp_args))\n", "jobs.append(submit(scripts['SVG to PDF'] % agp_gg_age_args))\n", "jobs.append(submit(scripts['SVG to PDF'] % agp_gg_country_args))\n", "res = wait_on(jobs)" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next, we're going to create the macros file for the resulting template. " ] }, { "cell_type": "code", "collapsed": false, "input": [ "macros_fp = get_relative_new_path('mod1_macros.tex')\n", "\n", "# get the date and format for human consumption (e.g., January 1, 2014)\n", "cur_date = datetime.datetime.now()\n", "date_fmt = cur_date.strftime(\"%B %d, %Y\")\n", "\n", "# format the counts into comma separated (e.g., 1,234,567)\n", "agp_samples_fmt = locale.format(\"%d\", agp_sample_count, grouping=True)\n", "agp_participants_fmt = locale.format(\"%d\", agp_unique_participants, grouping=True)\n", "agp_sequences_fmt = locale.format(\"%d\", agp_seq_count, grouping=True)\n", "pgp_samples_fmt = locale.format(\"%d\", pgp_sample_count, grouping=True)\n", "pgp_participants_fmt = locale.format(\"%d\", pgp_unique_participants, grouping=True)\n", "pgp_sequences_fmt = locale.format(\"%d\", pgp_seq_count, grouping=True)\n", "\n", "# build the macro template for the latex document\n", "macro_template = [\"% release date\"]\n", "macro_template.append(\"\\def\\\\releaseDate{%s}\" % date_fmt)\n", "macro_template.append(\"% participants paragraph\")\n", "macro_template.append(\"\\def\\\\numSamples{%s}\" % agp_samples_fmt)\n", "macro_template.append(\"\\def\\\\numParticipants{%s}\" % agp_participants_fmt)\n", "\n", "# published studies\n", "macro_template.append(\"% participants table\")\n", "macro_template.append(\"\\def\\hmpAge{Adults}\")\n", "macro_template.append(\"\\def\\hmpLocation{USA}\")\n", "macro_template.append(\"\\def\\hmpSamples{4,788}\")\n", "macro_template.append(\"\\def\\hmpParticipants{242}\")\n", "macro_template.append(\"\\def\\hmpSequences{36,797,226}\")\n", "macro_template.append(\"\\def\\ggAge{Adults,Children}\")\n", "macro_template.append(\"\\def\\ggLocation{Venezuela, Malawi, USA}\")\n", "macro_template.append(\"\\def\\ggSamples{531}\")\n", "macro_template.append(\"\\def\\ggParticipants{531}\")\n", "macro_template.append(\"\\def\\ggSequences{1,093,740,274}\")\n", "\n", "# growing studies\n", "macro_template.append(\"\\def\\pgpAge{Adults}\")\n", "macro_template.append(\"\\def\\pgpLocation{USA}\")\n", "macro_template.append(\"\\def\\pgpSamples{%s}\" % pgp_samples_fmt)\n", "macro_template.append(\"\\def\\pgpParticipants{%s}\" % pgp_participants_fmt)\n", "macro_template.append(\"\\def\\pgpSequences{%s}\" % pgp_sequences_fmt)\n", "macro_template.append(\"\\def\\\\agpAge{Adults, Children}\")\n", "macro_template.append(\"\\def\\\\agpLocation{Global}\")\n", "macro_template.append(\"\\def\\\\agpSamples{%s}\" % agp_samples_fmt)\n", "macro_template.append(\"\\def\\\\agpParticipants{%s}\" % agp_participants_fmt)\n", "macro_template.append(\"\\def\\\\agpSequences{%s}\" % agp_sequences_fmt)\n", "\n", "macro_template.append(\"% diversity figure\")\n", "macro_template.append(\"\\def\\\\numParticipantsLowerEstimate{%s}\" % agp_participants_fmt)\n", "\n", "macros = open(macros_fp, 'w')\n", "macros.write('\\n'.join(macro_template))\n", "macros.write('\\n')\n", "macros.close()" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Finally, we'll actually populate the template." ] }, { "cell_type": "code", "collapsed": false, "input": [ "# summary static images\n", "for_mod1 = ['ag_plots_fecal_legend.pdf',\n", " 'ag_plots_fecal_stack.pdf',\n", " 'ag_plots_oral_stack.pdf',\n", " 'ag_plots_skin_stack.pdf',\n", " 'agp_gg_age.pdf',\n", " 'agp_gg_country.pdf',\n", " 'agp_hmp.pdf',\n", " 'agp_pgp.pdf',\n", " 'fig2.pdf',\n", " 'fig3.pdf',\n", " 'fig5.pdf',\n", " 'fig6_a.pdf',\n", " 'fig6_b.pdf',\n", " 'fig6_legend.pdf',\n", " 's1.pdf',\n", " 's2.pdf',\n", " 's3.pdf',\n", " 'logoshape.pdf',\n", " 'legend_age.pdf',\n", " 'legend_agp_gg.pdf',\n", " 'legend_agp_hmp_pgp.pdf'\n", " ]\n", "\n", "os.mkdir(get_relative_new_path('pdfs-mod1'))\n", "for f in for_mod1:\n", " shutil.copy(get_relative_existing_path(f), get_relative_new_path('pdfs-mod1/'))\n", " \n", "res = wait_on(submit(scripts['To PDF'] % {'path': working_dir, 'input': 'mod1_main.tex'}))" ], "language": "python", "metadata": {}, "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And we're done!" ] } ], "metadata": {} } ] }