{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial\n",
    "\n",
    "I assume you've already installed refgenie. In this tutorial I'll show you a few ways to use refgenie from the command line (commands that start with a `!`), and also some Python commands.\n",
    "\n",
    "To start, initialize an empty refgenie configuration file from the shell and subscribe to the desired asset server:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized genome configuration file: /Users/mstolarczyk/code/refgenie/docs_jupyter/refgenie.yaml\r\n",
      "Created directories:\r\n",
      " - /Users/mstolarczyk/code/refgenie/docs_jupyter/data\r\n",
      " - /Users/mstolarczyk/code/refgenie/docs_jupyter/alias\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie init -c refgenie.yaml -s http://rg.databio.org"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's what it looks like:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config_version: 0.4\r\n",
      "genome_folder: /Users/mstolarczyk/code/refgenie/docs_jupyter\r\n",
      "genome_servers: \r\n",
      " - http://rg.databio.org\r\n",
      "genomes: null\r\n"
     ]
    }
   ],
   "source": [
    "!cat refgenie.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[3m                             Remote refgenie assets                             \u001b[0m\r\n",
      "\u001b[3m                       Server URL: http://rg.databio.org                        \u001b[0m\r\n",
      "┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\r\n",
      "┃\u001b[1m \u001b[0m\u001b[1mgenome          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1massets                                                   \u001b[0m\u001b[1m \u001b[0m┃\r\n",
      "┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\r\n",
      "│ rCRSd            │ fasta, bowtie2_index, bwa_index, hisat2_index,            │\r\n",
      "│                  │ star_index, bismark_bt2_index                             │\r\n",
      "│ hg18_cdna        │ fasta, kallisto_index                                     │\r\n",
      "│ hs38d1           │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ hg38_cdna        │ fasta, kallisto_index, salmon_index                       │\r\n",
      "│ human_repeats    │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ rn6_cdna         │ fasta, kallisto_index, salmon_index                       │\r\n",
      "│ mm10_cdna        │ fasta, kallisto_index, salmon_index                       │\r\n",
      "│ hg38_chr22       │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ hg38             │ fasta, gencode_gtf, ensembl_gtf, refgene_anno,            │\r\n",
      "│                  │ fasta_txome, ensembl_rb, feat_annotation,                 │\r\n",
      "│                  │ suffixerator_index, cellranger_reference, bowtie2_index,  │\r\n",
      "│                  │ bwa_index, tallymer_index, hisat2_index, star_index,      │\r\n",
      "│                  │ bismark_bt2_index, salmon_partial_sa_index                │\r\n",
      "│ hg19_cdna        │ fasta, kallisto_index, salmon_index                       │\r\n",
      "│ human_rDNA       │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ human_alu        │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, bismark_bt2_index           │\r\n",
      "│ human_alphasat   │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ mouse_chrM2x     │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ t7               │ fasta, bowtie2_index                                      │\r\n",
      "│ mm10             │ fasta, gencode_gtf, ensembl_gtf, refgene_anno,            │\r\n",
      "│                  │ fasta_txome, ensembl_rb, feat_annotation,                 │\r\n",
      "│                  │ suffixerator_index, cellranger_reference, bwa_index,      │\r\n",
      "│                  │ bowtie2_index, hisat2_index, tallymer_index, star_index,  │\r\n",
      "│                  │ bismark_bt2_index, salmon_partial_sa_index                │\r\n",
      "│ dm6              │ fasta, gencode_gtf, ensembl_gtf, refgene_anno,            │\r\n",
      "│                  │ bowtie2_index                                             │\r\n",
      "│ hg18             │ fasta, gencode_gtf, fasta_txome, suffixerator_index,      │\r\n",
      "│                  │ cellranger_reference, bwa_index, bowtie2_index,           │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index                                         │\r\n",
      "│ hg19             │ fasta, gencode_gtf, ensembl_gtf, refgene_anno,            │\r\n",
      "│                  │ fasta_txome, ensembl_rb, feat_annotation,                 │\r\n",
      "│                  │ suffixerator_index, cellranger_reference, bwa_index,      │\r\n",
      "│                  │ bowtie2_index, tallymer_index, hisat2_index, star_index,  │\r\n",
      "│                  │ salmon_partial_sa_index, bismark_bt2_index                │\r\n",
      "│ rn6              │ fasta, ensembl_gtf, refgene_anno, fasta_txome,            │\r\n",
      "│                  │ suffixerator_index, bwa_index, bowtie2_index,             │\r\n",
      "│                  │ tallymer_index, hisat2_index, star_index,                 │\r\n",
      "│                  │ bismark_bt2_index, salmon_partial_sa_index                │\r\n",
      "│ hg38_noalt_decoy │ fasta, suffixerator_index, bowtie2_index, bwa_index,      │\r\n",
      "│                  │ tallymer_index, hisat2_index, bismark_bt2_index           │\r\n",
      "│ mm10_primary     │ fasta, bowtie2_index, bwa_index                           │\r\n",
      "│ hg38_primary     │ fasta, bowtie2_index, bwa_index                           │\r\n",
      "│ hg38_mm10        │ fasta, bwa_index                                          │\r\n",
      "└──────────────────┴───────────────────────────────────────────────────────────┘\r\n",
      "\u001b[2;3m             use refgenie listr -g <genome> for more detailed view              \u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie listr -c refgenie.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's enter python and do some stuff."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import refgenconf\n",
    "rgc = refgenconf.RefGenConf(filepath=\"refgenie.yaml\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `pull` to download one of the assets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abab7f40d9654ca6ba8c60471cbe303a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(['43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a', 'fasta', 'default'],\n",
       " {'asset_path': 'fasta',\n",
       "  'asset_digest': '8dfe402f7d29d5b036dd8937119e4404',\n",
       "  'archive_digest': 'bfb7877ee114c61a17a50bd471de47a2',\n",
       "  'asset_size': '39.4KB',\n",
       "  'archive_size': '9.1KB',\n",
       "  'seek_keys': {'fasta': '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a.fa',\n",
       "   'fai': '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a.fa.fai',\n",
       "   'chrom_sizes': '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a.chrom.sizes'},\n",
       "  'asset_parents': [],\n",
       "  'asset_children': ['43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/suffixerator_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/bowtie2_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/bwa_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/tallymer_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/hisat2_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/star_index:default',\n",
       "   '43f14ba8beed34d52edb244e26f193df6edbb467bd55d37a/bismark_bt2_index:default']},\n",
       " 'http://rg.databio.org')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgc.pull(\"mouse_chrM2x\", \"fasta\", \"default\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once it's downloaded, use `seek` to retrieve a path to it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/mstolarczyk/code/refgenie/docs_jupyter/alias/mouse_chrM2x/fasta/default/mouse_chrM2x.fa'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgc.seek(\"mouse_chrM2x\", \"fasta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can get the unique asset identifier with `id()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8dfe402f7d29d5b036dd8937119e4404'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgc.id(\"mouse_chrM2x\", \"fasta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building and pulling from the command line"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we can build a fasta asset instead of pulling one. Back to the shell, we'll grab the Revised Cambridge Reference Sequence (human mitochondrial genome, because it's small):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-03-09 12:22:40--  http://big.databio.org/refgenie_raw/files.rCRSd.fasta.fasta\n",
      "Resolving big.databio.org (big.databio.org)... 128.143.245.181, 128.143.245.182\n",
      "Connecting to big.databio.org (big.databio.org)|128.143.245.181|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 8399 (8.2K) [application/octet-stream]\n",
      "Saving to: ‘rCRSd.fa.gz’\n",
      "\n",
      "rCRSd.fa.gz         100%[===================>]   8.20K  --.-KB/s    in 0.006s  \n",
      "\n",
      "2021-03-09 12:22:40 (1.35 MB/s) - ‘rCRSd.fa.gz’ saved [8399/8399]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget -O rCRSd.fa.gz http://big.databio.org/refgenie_raw/files.rCRSd.fasta.fasta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 'default' as the default tag for 'rCRSd/fasta'\n",
      "Recipe validated successfully against a schema: /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/refgenie/schemas/recipe_schema.yaml\n",
      "Building 'rCRSd/fasta:default' using 'fasta' recipe\n",
      "Initializing genome: rCRSd\n",
      "Loaded AnnotatedSequenceDigestList (1 sequences)\n",
      "Set genome alias (94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4: rCRSd)\n",
      "Created alias directories: \n",
      " - /Users/mstolarczyk/code/refgenie/docs_jupyter/alias/rCRSd\n",
      "Saving outputs to:\n",
      "- content: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4\n",
      "- logs: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/_refgenie_build\n",
      "### Pipeline run code and environment:\n",
      "\n",
      "*              Command:  `/Library/Frameworks/Python.framework/Versions/3.6/bin/refgenie build rCRSd/fasta -c refgenie.yaml --files fasta=rCRSd.fa.gz -R`\n",
      "*         Compute host:  MichalsMBP\n",
      "*          Working dir:  /Users/mstolarczyk/code/refgenie/docs_jupyter\n",
      "*            Outfolder:  /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/_refgenie_build/\n",
      "*  Pipeline started at:   (03-09 12:22:41) elapsed: 0.0 _TIME_\n",
      "\n",
      "### Version log:\n",
      "\n",
      "*       Python version:  3.6.5\n",
      "*          Pypiper dir:  `/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pypiper`\n",
      "*      Pypiper version:  0.12.1\n",
      "*         Pipeline dir:  `/Library/Frameworks/Python.framework/Versions/3.6/bin`\n",
      "*     Pipeline version:  None\n",
      "\n",
      "### Arguments passed to pipeline:\n",
      "\n",
      "* `asset_registry_paths`:  `['rCRSd/fasta']`\n",
      "*             `assets`:  `None`\n",
      "*            `command`:  `build`\n",
      "*        `config_file`:  `refgenie.yaml`\n",
      "*             `docker`:  `False`\n",
      "*              `files`:  `[['fasta=rCRSd.fa.gz']]`\n",
      "*             `genome`:  `None`\n",
      "*      `genome_config`:  `refgenie.yaml`\n",
      "* `genome_description`:  `None`\n",
      "*             `logdev`:  `False`\n",
      "*          `new_start`:  `False`\n",
      "*          `outfolder`:  `/Users/mstolarczyk/code/refgenie/docs_jupyter/data`\n",
      "*             `params`:  `None`\n",
      "*             `recipe`:  `None`\n",
      "*            `recover`:  `True`\n",
      "*       `requirements`:  `False`\n",
      "*             `silent`:  `False`\n",
      "*     `skip_read_lock`:  `False`\n",
      "*    `tag_description`:  `None`\n",
      "*          `verbosity`:  `None`\n",
      "*            `volumes`:  `None`\n",
      "\n",
      "----------------------------------------\n",
      "\n",
      "Target to produce: `/Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/_refgenie_build/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4_fasta__default.flag`  \n",
      "\n",
      "> `cp rCRSd.fa.gz /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa.gz` (63575)\n",
      "<pre>\n",
      "psutil.ZombieProcess process still exists but it's a zombie (pid=63575)\n",
      "Warning: couldn't add memory use for process: 63575\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0GB.  \n",
      "  PID: 63575;\tCommand: cp;\tReturn code: 0;\tMemory used: 0GB\n",
      "\n",
      "\n",
      "> `gzip -df /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa.gz` (63576)\n",
      "<pre>\n",
      "psutil.ZombieProcess process still exists but it's a zombie (pid=63576)\n",
      "Warning: couldn't add memory use for process: 63576\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0GB.  \n",
      "  PID: 63576;\tCommand: gzip;\tReturn code: 0;\tMemory used: 0GB\n",
      "\n",
      "\n",
      "> `samtools faidx /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa` (63577)\n",
      "<pre>\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0.001GB.  \n",
      "  PID: 63577;\tCommand: samtools;\tReturn code: 0;\tMemory used: 0.001GB\n",
      "\n",
      "\n",
      "> `cut -f 1,2 /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa.fai > /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.chrom.sizes` (63578)\n",
      "<pre>\n",
      "psutil.ZombieProcess process still exists but it's a zombie (pid=63578)\n",
      "Warning: couldn't add memory use for process: 63578\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0.001GB.  \n",
      "  PID: 63578;\tCommand: cut;\tReturn code: 0;\tMemory used: 0GB\n",
      "\n",
      "\n",
      "> `touch /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/_refgenie_build/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4_fasta__default.flag` (63580)\n",
      "<pre>\n",
      "psutil.ZombieProcess process still exists but it's a zombie (pid=63580)\n",
      "Warning: couldn't add memory use for process: 63580\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0.001GB.  \n",
      "  PID: 63580;\tCommand: touch;\tReturn code: 0;\tMemory used: 0GB\n",
      "\n",
      "Asset digest: 4eb430296bc02ed7e4006624f1d5ac53\n",
      "Default tag for '94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta' set to: default\n",
      "\n",
      "### Pipeline completed. Epilogue\n",
      "*        Elapsed time (this run):  0:00:00\n",
      "*  Total elapsed time (all runs):  0:00:00\n",
      "*         Peak memory (this run):  0.0015 GB\n",
      "*        Pipeline completed time: 2021-03-09 12:22:41\n",
      "Finished building 'fasta' asset\n",
      "Created alias directories: \n",
      " - /Users/mstolarczyk/code/refgenie/docs_jupyter/alias/rCRSd/fasta/default\n"
     ]
    }
   ],
   "source": [
    "!refgenie build rCRSd/fasta -c refgenie.yaml  --files fasta=rCRSd.fa.gz -R"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The asset should be available for local use, let's call `refgenie list` to check it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[3m                        Local refgenie assets                         \u001b[0m\r\n",
      "\u001b[3m             Server subscriptions: http://rg.databio.org              \u001b[0m\r\n",
      "┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓\r\n",
      "┃\u001b[1m \u001b[0m\u001b[1mgenome   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1masset (\u001b[0m\u001b[1;3mseek_keys\u001b[0m\u001b[1m)                         \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mtags     \u001b[0m\u001b[1m \u001b[0m┃\r\n",
      "┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩\r\n",
      "│ rCRSd     │ fasta (\u001b[3mfasta, fai, chrom_sizes\u001b[0m)            │ default   │\r\n",
      "└───────────┴────────────────────────────────────────────┴───────────┘\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie list -c refgenie.yaml --genome rCRSd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can retrieve the path to this asset with:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/mstolarczyk/code/refgenie/docs_jupyter/alias/rCRSd/fasta/default/rCRSd.fa\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie seek rCRSd/fasta -c refgenie.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Naturally, we can do the same thing from within Python:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/mstolarczyk/code/refgenie/docs_jupyter/alias/rCRSd/fasta/default/rCRSd.fa'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgc = refgenconf.RefGenConf(\"refgenie.yaml\")\n",
    "rgc.seek(\"rCRSd\", \"fasta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, if we have bowtie2-build in our `$PATH` we can build the `bowtie2_index` asset with no further requirements.\n",
    "\n",
    "Let's check the requirements with `refgenie build --requirements`:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'bowtie2_index' recipe requirements: \r\n",
      "- assets:\r\n",
      "\tfasta (fasta asset for genome); default: fasta\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie build rCRSd/bowtie2_index -c refgenie.yaml --requirements"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since I already have the fasta asset, that means I don't need anything else to build the bowtie2_index."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 'default' as the default tag for 'rCRSd/bowtie2_index'\n",
      "Recipe validated successfully against a schema: /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/refgenie/schemas/recipe_schema.yaml\n",
      "Building 'rCRSd/bowtie2_index:default' using 'bowtie2_index' recipe\n",
      "Saving outputs to:\n",
      "- content: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4\n",
      "- logs: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/_refgenie_build\n",
      "### Pipeline run code and environment:\n",
      "\n",
      "*              Command:  `/Library/Frameworks/Python.framework/Versions/3.6/bin/refgenie build rCRSd/bowtie2_index -c refgenie.yaml`\n",
      "*         Compute host:  MichalsMBP\n",
      "*          Working dir:  /Users/mstolarczyk/code/refgenie/docs_jupyter\n",
      "*            Outfolder:  /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/_refgenie_build/\n",
      "*  Pipeline started at:   (03-09 12:22:45) elapsed: 0.0 _TIME_\n",
      "\n",
      "### Version log:\n",
      "\n",
      "*       Python version:  3.6.5\n",
      "*          Pypiper dir:  `/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pypiper`\n",
      "*      Pypiper version:  0.12.1\n",
      "*         Pipeline dir:  `/Library/Frameworks/Python.framework/Versions/3.6/bin`\n",
      "*     Pipeline version:  None\n",
      "\n",
      "### Arguments passed to pipeline:\n",
      "\n",
      "* `asset_registry_paths`:  `['rCRSd/bowtie2_index']`\n",
      "*             `assets`:  `None`\n",
      "*            `command`:  `build`\n",
      "*        `config_file`:  `refgenie.yaml`\n",
      "*             `docker`:  `False`\n",
      "*              `files`:  `None`\n",
      "*             `genome`:  `None`\n",
      "*      `genome_config`:  `refgenie.yaml`\n",
      "* `genome_description`:  `None`\n",
      "*             `logdev`:  `False`\n",
      "*          `new_start`:  `False`\n",
      "*          `outfolder`:  `/Users/mstolarczyk/code/refgenie/docs_jupyter/data`\n",
      "*             `params`:  `None`\n",
      "*             `recipe`:  `None`\n",
      "*            `recover`:  `False`\n",
      "*       `requirements`:  `False`\n",
      "*             `silent`:  `False`\n",
      "*     `skip_read_lock`:  `False`\n",
      "*    `tag_description`:  `None`\n",
      "*          `verbosity`:  `None`\n",
      "*            `volumes`:  `None`\n",
      "\n",
      "----------------------------------------\n",
      "\n",
      "Target to produce: `/Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/_refgenie_build/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4_bowtie2_index__default.flag`  \n",
      "\n",
      "> `bowtie2-build /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4` (63609)\n",
      "<pre>\n",
      "Settings:\n",
      "  Output files: \"/Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.*.bt2\"\n",
      "  Line rate: 6 (line is 64 bytes)\n",
      "  Lines per side: 1 (side is 64 bytes)\n",
      "  Offset rate: 4 (one in 16)\n",
      "  FTable chars: 10\n",
      "  Strings: unpacked\n",
      "  Max bucket size: default\n",
      "  Max bucket size, sqrt multiplier: default\n",
      "  Max bucket size, len divisor: 4\n",
      "  Difference-cover sample period: 1024\n",
      "  Endianness: little\n",
      "  Actual local endianness: little\n",
      "  Sanity checking: disabled\n",
      "  Assertions: disabled\n",
      "  Random seed: 0\n",
      "  Sizeofs: void*:8, int:4, long:8, size_t:8\n",
      "Input files DNA, FASTA:\n",
      "  /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/fasta/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.fa\n",
      "Building a SMALL index\n",
      "Reading reference sizes\n",
      "  Time reading reference sizes: 00:00:00\n",
      "Calculating joined length\n",
      "Writing header\n",
      "Reserving space for joined string\n",
      "Joining reference sequences\n",
      "  Time to join reference sequences: 00:00:00\n",
      "bmax according to bmaxDivN setting: 8284\n",
      "Using parameters --bmax 6213 --dcv 1024\n",
      "  Doing ahead-of-time memory usage test\n",
      "  Passed!  Constructing with these parameters: --bmax 6213 --dcv 1024\n",
      "Constructing suffix-array element generator\n",
      "Building DifferenceCoverSample\n",
      "  Building sPrime\n",
      "  Building sPrimeOrder\n",
      "  V-Sorting samples\n",
      "  V-Sorting samples time: 00:00:00\n",
      "  Allocating rank array\n",
      "  Ranking v-sort output\n",
      "  Ranking v-sort output time: 00:00:00\n",
      "  Invoking Larsson-Sadakane on ranks\n",
      "  Invoking Larsson-Sadakane on ranks time: 00:00:00\n",
      "  Sanity-checking and returning\n",
      "Building samples\n",
      "Reserving space for 12 sample suffixes\n",
      "Generating random suffixes\n",
      "QSorting 12 sample offsets, eliminating duplicates\n",
      "QSorting sample offsets, eliminating duplicates time: 00:00:00\n",
      "Multikey QSorting 12 samples\n",
      "  (Using difference cover)\n",
      "  Multikey QSorting samples time: 00:00:00\n",
      "Calculating bucket sizes\n",
      "Splitting and merging\n",
      "  Splitting and merging time: 00:00:00\n",
      "Avg bucket size: 33136 (target: 6212)\n",
      "Converting suffix-array elements to index image\n",
      "Allocating ftab, absorbFtab\n",
      "Entering Ebwt loop\n",
      "Getting block 1 of 1\n",
      "  No samples; assembling all-inclusive block\n",
      "  Sorting block of length 33136 for bucket 1\n",
      "  (Using difference cover)\n",
      "  Sorting block time: 00:00:00\n",
      "Returning block of 33137 for bucket 1\n",
      "Exited Ebwt loop\n",
      "fchr[A]: 0\n",
      "fchr[C]: 10248\n",
      "fchr[G]: 20610\n",
      "fchr[T]: 24948\n",
      "fchr[$]: 33136\n",
      "Exiting Ebwt::buildToDisk()\n",
      "Returning from initFromVector\n",
      "Wrote 4205567 bytes to primary EBWT file: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.1.bt2\n",
      "Wrote 8292 bytes to secondary EBWT file: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.2.bt2\n",
      "Re-opening _in1 and _in2 as input streams\n",
      "Returning from Ebwt constructor\n",
      "Headers:\n",
      "    len: 33136\n",
      "    bwtLen: 33137\n",
      "    sz: 8284\n",
      "    bwtSz: 8285\n",
      "    lineRate: 6\n",
      "    offRate: 4\n",
      "    offMask: 0xfffffff0\n",
      "    ftabChars: 10\n",
      "    eftabLen: 20\n",
      "    eftabSz: 80\n",
      "    ftabLen: 1048577\n",
      "    ftabSz: 4194308\n",
      "    offsLen: 2072\n",
      "    offsSz: 8288\n",
      "    lineSz: 64\n",
      "    sideSz: 64\n",
      "    sideBwtSz: 48\n",
      "    sideBwtLen: 192\n",
      "    numSides: 173\n",
      "    numLines: 173\n",
      "    ebwtTotLen: 11072\n",
      "    ebwtTotSz: 11072\n",
      "    color: 0\n",
      "    reverse: 0\n",
      "Total time for call to driver() for forward index: 00:00:00\n",
      "Reading reference sizes\n",
      "  Time reading reference sizes: 00:00:00\n",
      "Calculating joined length\n",
      "Writing header\n",
      "Reserving space for joined string\n",
      "Joining reference sequences\n",
      "  Time to join reference sequences: 00:00:00\n",
      "  Time to reverse reference sequence: 00:00:00\n",
      "bmax according to bmaxDivN setting: 8284\n",
      "Using parameters --bmax 6213 --dcv 1024\n",
      "  Doing ahead-of-time memory usage test\n",
      "  Passed!  Constructing with these parameters: --bmax 6213 --dcv 1024\n",
      "Constructing suffix-array element generator\n",
      "Building DifferenceCoverSample\n",
      "  Building sPrime\n",
      "  Building sPrimeOrder\n",
      "  V-Sorting samples\n",
      "  V-Sorting samples time: 00:00:00\n",
      "  Allocating rank array\n",
      "  Ranking v-sort output\n",
      "  Ranking v-sort output time: 00:00:00\n",
      "  Invoking Larsson-Sadakane on ranks\n",
      "  Invoking Larsson-Sadakane on ranks time: 00:00:00\n",
      "  Sanity-checking and returning\n",
      "Building samples\n",
      "Reserving space for 12 sample suffixes\n",
      "Generating random suffixes\n",
      "QSorting 12 sample offsets, eliminating duplicates\n",
      "QSorting sample offsets, eliminating duplicates time: 00:00:00\n",
      "Multikey QSorting 12 samples\n",
      "  (Using difference cover)\n",
      "  Multikey QSorting samples time: 00:00:00\n",
      "Calculating bucket sizes\n",
      "Splitting and merging\n",
      "  Splitting and merging time: 00:00:00\n",
      "Avg bucket size: 33136 (target: 6212)\n",
      "Converting suffix-array elements to index image\n",
      "Allocating ftab, absorbFtab\n",
      "Entering Ebwt loop\n",
      "Getting block 1 of 1\n",
      "  No samples; assembling all-inclusive block\n",
      "  Sorting block of length 33136 for bucket 1\n",
      "  (Using difference cover)\n",
      "  Sorting block time: 00:00:00\n",
      "Returning block of 33137 for bucket 1\n",
      "Exited Ebwt loop\n",
      "fchr[A]: 0\n",
      "fchr[C]: 10248\n",
      "fchr[G]: 20610\n",
      "fchr[T]: 24948\n",
      "fchr[$]: 33136\n",
      "Exiting Ebwt::buildToDisk()\n",
      "Returning from initFromVector\n",
      "Wrote 4205567 bytes to primary EBWT file: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.rev.1.bt2\n",
      "Wrote 8292 bytes to secondary EBWT file: /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4.rev.2.bt2\n",
      "Re-opening _in1 and _in2 as input streams\n",
      "Returning from Ebwt constructor\n",
      "Headers:\n",
      "    len: 33136\n",
      "    bwtLen: 33137\n",
      "    sz: 8284\n",
      "    bwtSz: 8285\n",
      "    lineRate: 6\n",
      "    offRate: 4\n",
      "    offMask: 0xfffffff0\n",
      "    ftabChars: 10\n",
      "    eftabLen: 20\n",
      "    eftabSz: 80\n",
      "    ftabLen: 1048577\n",
      "    ftabSz: 4194308\n",
      "    offsLen: 2072\n",
      "    offsSz: 8288\n",
      "    lineSz: 64\n",
      "    sideSz: 64\n",
      "    sideBwtSz: 48\n",
      "    sideBwtLen: 192\n",
      "    numSides: 173\n",
      "    numLines: 173\n",
      "    ebwtTotLen: 11072\n",
      "    ebwtTotSz: 11072\n",
      "    color: 0\n",
      "    reverse: 1\n",
      "Total time for backward call to driver() for mirror index: 00:00:00\n",
      "</pre>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0.003GB.  \n",
      "  PID: 63609;\tCommand: bowtie2-build;\tReturn code: 0;\tMemory used: 0.003GB\n",
      "\n",
      "\n",
      "> `touch /Users/mstolarczyk/code/refgenie/docs_jupyter/data/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index/default/_refgenie_build/94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4_bowtie2_index__default.flag` (63611)\n",
      "<pre>\n",
      "psutil.ZombieProcess process still exists but it's a zombie (pid=63611)\n",
      "Warning: couldn't add memory use for process: 63611\n",
      "</pre>\n",
      "Command completed. Elapsed time: 0:00:00. Running peak memory: 0.003GB.  \n",
      "  PID: 63611;\tCommand: touch;\tReturn code: 0;\tMemory used: 0GB\n",
      "\n",
      "Asset digest: 1262e30d4a87db9365d501de8559b3b4\n",
      "Default tag for '94e0d21feb576e6af61cd2a798ad30682ef2428bb7eabbb4/bowtie2_index' set to: default\n",
      "\n",
      "### Pipeline completed. Epilogue\n",
      "*        Elapsed time (this run):  0:00:01\n",
      "*  Total elapsed time (all runs):  0:00:00\n",
      "*         Peak memory (this run):  0.0028 GB\n",
      "*        Pipeline completed time: 2021-03-09 12:22:46\n",
      "Finished building 'bowtie2_index' asset\n",
      "Created alias directories: \n",
      " - /Users/mstolarczyk/code/refgenie/docs_jupyter/alias/rCRSd/bowtie2_index/default\n"
     ]
    }
   ],
   "source": [
    "!refgenie build rCRSd/bowtie2_index -c refgenie.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see a list of available recipes like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bismark_bt1_index, bismark_bt2_index, blacklist, bowtie2_index, bwa_index, cellranger_reference, dbnsfp, dbsnp, ensembl_gtf, ensembl_rb, epilog_index, fasta, fasta_txome, feat_annotation, gencode_gtf, hisat2_index, kallisto_index, refgene_anno, salmon_index, salmon_partial_sa_index, salmon_sa_index, star_index, suffixerator_index, tallymer_index, tgMap\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie list -c refgenie.yaml --recipes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can get the unique digest for any asset with `refgenie id`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4eb430296bc02ed7e4006624f1d5ac53\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie id rCRSd/fasta -c refgenie.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.6.5'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from platform import python_version \n",
    "python_version()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "refgenie 0.10.0-dev | refgenconf 0.10.0-dev\r\n"
     ]
    }
   ],
   "source": [
    "!refgenie --version"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}