{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"TODAY'S DATE:\n",
"Thu Nov 30 14:09:12 PST 2017\n",
"------------\n",
"\n",
"Distributor ID:\tUbuntu\n",
"Description:\tUbuntu 16.04.3 LTS\n",
"Release:\t16.04\n",
"Codename:\txenial\n",
"\n",
"------------\n",
"HOSTNAME: \n",
"emu\n",
"\n",
"------------\n",
"Computer Specs:\n",
"\n",
"Architecture: x86_64\n",
"CPU op-mode(s): 32-bit, 64-bit\n",
"Byte Order: Little Endian\n",
"CPU(s): 16\n",
"On-line CPU(s) list: 0-15\n",
"Thread(s) per core: 2\n",
"Core(s) per socket: 4\n",
"Socket(s): 2\n",
"NUMA node(s): 1\n",
"Vendor ID: GenuineIntel\n",
"CPU family: 6\n",
"Model: 26\n",
"Model name: Intel(R) Xeon(R) CPU E5520 @ 2.27GHz\n",
"Stepping: 5\n",
"CPU MHz: 2394.000\n",
"CPU max MHz: 2394.0000\n",
"CPU min MHz: 1596.0000\n",
"BogoMIPS: 4521.80\n",
"Virtualization: VT-x\n",
"L1d cache: 32K\n",
"L1i cache: 32K\n",
"L2 cache: 256K\n",
"L3 cache: 8192K\n",
"NUMA node0 CPU(s): 0-15\n",
"Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm dca sse4_1 sse4_2 popcnt lahf_lm tpr_shadow vnmi flexpriority ept vpid dtherm ida\n",
"\n",
"------------\n",
"\n",
"Memory Specs\n",
"\n",
" total used free shared buff/cache available\n",
"Mem: 47G 7.6G 29G 575M 9.8G 38G\n",
"Swap: 11G 5.6G 6.4G\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"No LSB modules are available.\n"
]
}
],
"source": [
"%%bash\n",
"echo \"TODAY'S DATE:\"\n",
"date\n",
"echo \"------------\"\n",
"echo \"\"\n",
"lsb_release -a\n",
"echo \"\"\n",
"echo \"------------\"\n",
"echo \"HOSTNAME: \"; hostname \n",
"echo \"\"\n",
"echo \"------------\"\n",
"echo \"Computer Specs:\"\n",
"echo \"\"\n",
"lscpu\n",
"echo \"\"\n",
"echo \"------------\"\n",
"echo \"\"\n",
"echo \"Memory Specs\"\n",
"echo \"\"\n",
"free -mh"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%%bash\n",
"mkdir /home/sam/analyses/20171130_oly_pbjelly"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[0m\u001b[01;34massembly\u001b[0m/ jelly.out.qual output.err \u001b[01;34msupport\u001b[0m/\r\n",
"gap_fill_status.txt liftOverTable.json output.out\r\n",
"jelly.out.fasta \u001b[01;34mmapping\u001b[0m/ Protocol.xml\r\n"
]
}
],
"source": [
"ls /home/sam/analyses/20171120_oly_pbjelly/"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"ls /home/sam/analyses/20171130_oly_pbjelly/"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%%bash\n",
"cp /home/sam/analyses/20171120_oly_pbjelly/Protocol.xml /home/sam/analyses/20171130_oly_pbjelly/"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<jellyProtocol>\n",
" <reference>/home/sam/data/oly_BGI_scaffolds.fasta</reference> \n",
" <outputDir>/home/sam/analyses/20171120_oly_pbjelly</outputDir>\n",
" <blasr>-minMatch 8 -minPctIdentity 70 -bestn 1 -nCandidates 20 -maxScore -500 -nproc 48 -noSplitSubreads</blasr>\n",
" <input baseDir=\"/home/sam/data/\">\n",
" <job>m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq</job>\n",
" <job>m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq</job>\n",
" </input>\n",
"</jellyProtocol>\n"
]
}
],
"source": [
"%%bash\n",
"cat /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%%bash\n",
"sed -i 's/20171120_oly_pbjelly/20171130_oly_pbjelly/g' /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<jellyProtocol>\n",
" <reference>/home/sam/data/oly_BGI_scaffolds.fasta</reference> \n",
" <outputDir>/home/sam/analyses/20171130_oly_pbjelly</outputDir>\n",
" <blasr>-minMatch 8 -minPctIdentity 70 -bestn 1 -nCandidates 20 -maxScore -500 -nproc 48 -noSplitSubreads</blasr>\n",
" <input baseDir=\"/home/sam/data/\">\n",
" <job>m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq</job>\n",
" <job>m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq</job>\n",
"\t<job>m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq</job>\n",
" </input>\n",
"</jellyProtocol>\n"
]
}
],
"source": [
"%%bash\n",
"cat /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"source /home/shared/PBSuite_15.8.24/setup.sh\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py setup /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py mapping /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py support /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py extraction /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py assembly /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml\n",
"time python /home/shared/PBSuite_15.8.24/bin/Jelly.py output /home/sam/analyses/20171130_oly_pbjelly/Protocol.xml"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Well, Firefox crashed while the command above was running (20180103). However, PB Jelly is still running. I'll report on output once it has completed."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%%bash\n",
"source ~/.bashrc\n",
"source ~/.bash_aliases\n",
"emailme"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total 3.4G\n",
"-rw-rw-r-- 1 sam sam 1.4K Nov 30 14:13 Protocol.xml\n",
"drwxrwxr-x 2 sam sam 4.0K Dec 2 03:40 mapping\n",
"drwxrwxr-x 2 sam sam 4.0K Dec 2 05:26 support\n",
"drwxrwxr-x 1 sam sam 26M Dec 2 15:07 assembly\n",
"-rw-rw-r-- 1 sam sam 0 Dec 31 07:17 output.out\n",
"-rw-rw-r-- 1 sam sam 18M Dec 31 11:46 gap_fill_status.txt\n",
"-rw-rw-r-- 1 sam sam 1.2G Jan 9 04:43 jelly.out.qual\n",
"-rw-rw-r-- 1 sam sam 1.2G Jan 9 04:43 jelly.out.fasta\n",
"-rw-rw-r-- 1 sam sam 91M Jan 9 04:43 liftOverTable.json\n",
"-rw-rw-r-- 1 sam sam 862M Jan 9 04:43 output.err\n"
]
}
],
"source": [
"%%bash\n",
"ls -lthr /home/sam/analyses/20171130_oly_pbjelly/"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"696946\n"
]
}
],
"source": [
"%%bash\n",
"# Count the number of \">\" (i.e. scaffolds) in the output file\n",
"tr -cd \\> < /home/sam/analyses/20171130_oly_pbjelly/jelly.out.fasta | wc -c"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/Protocol.xml': Permission denied\n",
"cp: cannot create directory '/mnt/owl/Athaliana/20171130_oly_pbjelly/mapping': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/output.out': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/output.err': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/gap_fill_status.txt': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/jelly.out.fasta': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/jelly.out.qual': Permission denied\n",
"cp: cannot create regular file '/mnt/owl/Athaliana/20171130_oly_pbjelly/liftOverTable.json': Permission denied\n",
"cp: cannot create directory '/mnt/owl/Athaliana/20171130_oly_pbjelly/assembly': Permission denied\n",
"cp: cannot create directory '/mnt/owl/Athaliana/20171130_oly_pbjelly/support': Permission denied\n",
"cp: preserving permissions for '/mnt/owl/Athaliana/20171130_oly_pbjelly': Operation not permitted\n",
"\n",
"real\t0m1.368s\n",
"user\t0m0.000s\n",
"sys\t0m0.000s\n",
"bash: line 4: emailme: command not found\n"
]
}
],
"source": [
"%%bash\n",
"time cp -pR /home/sam/analyses/20171130_oly_pbjelly/ /mnt/owl/Athaliana/\n",
"source ~/.bashrc\n",
"source ~/.bash_aliases\n",
"emailme"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Used ```sudo``` outside of notebook for copying."
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total 3.4G\n",
"-rw-rw-r-- 1 sam users 1.4K Nov 30 14:13 Protocol.xml\n",
"drwxrwxr-x 2 sam sam 0 Dec 2 03:40 mapping\n",
"drwxrwxr-x 2 sam sam 0 Dec 2 05:26 support\n",
"drwxrwxr-x 1 sam sam 0 Dec 2 15:07 assembly\n",
"-rw-rw-r-- 1 sam users 0 Dec 31 07:17 output.out\n",
"-rw-rw-r-- 1 sam users 18M Dec 31 11:46 gap_fill_status.txt\n",
"-rw-rw-r-- 1 sam users 1.2G Jan 9 04:43 jelly.out.qual\n",
"-rw-rw-r-- 1 sam users 1.2G Jan 9 04:43 jelly.out.fasta\n",
"-rw-rw-r-- 1 sam users 91M Jan 9 04:43 liftOverTable.json\n",
"-rw-rw-r-- 1 sam users 862M Jan 9 04:43 output.err\n"
]
}
],
"source": [
"%%bash\n",
"ls -ltrh /mnt/owl/Athaliana/20171130_oly_pbjelly/"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"Started this notebook again to run Quast."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/home/sam/software/quast-4.5/quast.py -t 24 /mnt/owl/Athaliana/20171130_oly_pbjelly/jelly.out.fasta\n",
"\n",
"Version: 4.5, 15ca3b9\n",
"\n",
"System information:\n",
" OS: Linux-4.4.0-109-generic-x86_64-with-Ubuntu-16.04-xenial (linux_64)\n",
" Python version: 2.7.12\n",
" CPUs number: 24\n",
"\n",
"Started: 2018-01-16 09:35:09\n",
"\n",
"Logging to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/quast.log\n",
"\n",
"CWD: /home/sam/gitrepos/LabDocs/jupyter_nbs/sam\n",
"Main parameters: \n",
" Threads: 24, minimum contig length: 500, ambiguity: one, threshold for extensive misassembly size: 1000\n",
"\n",
"WARNING: Can't draw plots: python-matplotlib is missing or corrupted.\n",
"\n",
"Contigs:\n",
" Pre-processing...\n",
" /mnt/owl/Athaliana/20171130_oly_pbjelly/jelly.out.fasta ==> jelly.out\n",
"\n",
"2018-01-16 09:36:13\n",
"Running Basic statistics processor...\n",
" Contig files: \n",
" jelly.out\n",
" Calculating N50 and L50...\n",
" jelly.out, N50 = 12433, L50 = 26241, Total length = 1180563613, GC % = 36.57, # N's per 100 kbp = 6580.58\n",
"Done.\n",
"\n",
"NOTICE: Genes are not predicted by default. Use --gene-finding option to enable it.\n",
"\n",
"2018-01-16 09:37:14\n",
"Creating large visual summaries...\n",
"This may take a while: press Ctrl-C to skip this step..\n",
" 1 of 1: Creating Icarus viewers...\n",
"Done\n",
"\n",
"2018-01-16 09:37:32\n",
"RESULTS:\n",
" Text versions of total report are saved to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/report.txt, report.tsv, and report.tex\n",
" Text versions of transposed total report are saved to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/transposed_report.txt, transposed_report.tsv, and transposed_report.tex\n",
" HTML version (interactive tables and plots) saved to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/report.html\n",
" Icarus (contig browser) is saved to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/icarus.html\n",
" Log saved to /home/sam/gitrepos/LabDocs/jupyter_nbs/sam/quast_results/results_2018_01_16_09_35_09/quast.log\n",
"\n",
"Finished: 2018-01-16 09:37:32\n",
"Elapsed time: 0:02:23.101954\n",
"NOTICEs: 1; WARNINGs: 1; non-fatal ERRORs: 0\n",
"\n",
"Thank you for using QUAST!\n"
]
}
],
"source": [
"!python /home/sam/software/quast-4.5/quast.py \\\n",
"-t 24 \\\n",
"/mnt/owl/Athaliana/20171130_oly_pbjelly/jelly.out.fasta"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"All statistics are based on contigs of size >= 500 bp, unless otherwise noted (e.g., \"# contigs (>= 0 bp)\" and \"Total length (>= 0 bp)\" include all contigs).\n",
"Suggestion: assembly jelly.out contains continuous fragments of N's of length >= 10 bp. You may consider rerunning QUAST using --scaffolds (-s) option!\n",
"\n",
"Assembly jelly.out \n",
"# contigs (>= 0 bp) 696946 \n",
"# contigs (>= 1000 bp) 159429 \n",
"# contigs (>= 5000 bp) 68750 \n",
"# contigs (>= 10000 bp) 35320 \n",
"# contigs (>= 25000 bp) 7048 \n",
"# contigs (>= 50000 bp) 894 \n",
"Total length (>= 0 bp) 1253001795\n",
"Total length (>= 1000 bp) 1140787867\n",
"Total length (>= 5000 bp) 932263178 \n",
"Total length (>= 10000 bp) 691523275 \n",
"Total length (>= 25000 bp) 261425921 \n",
"Total length (>= 50000 bp) 57741906 \n",
"# contigs 213264 \n",
"Largest contig 194507 \n",
"Total length 1180563613\n",
"GC (%) 36.57 \n",
"N50 12433 \n",
"N75 5983 \n",
"L50 26241 \n",
"L75 60202 \n",
"# N's per 100 kbp 6580.58 \n"
]
}
],
"source": [
"%%bash\n",
"cat quast_results/results_2018_01_16_09_35_09/report.txt"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"cp: cannot create directory '/mnt/owl/Athaliana/quast_results/results_2018_01_16_09_35_09': Permission denied\n"
]
}
],
"source": [
"%%bash\n",
"cp -pR quast_results/results_2018_01_16_09_35_09/ /mnt/owl/Athaliana/quast_results/"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Needed to use ```sudo``` to cp, so did that outside of notebook."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"basic_stats\n",
"icarus.html\n",
"icarus_viewers\n",
"quast.log\n",
"report.html\n",
"report.tex\n",
"report.tsv\n",
"report.txt\n",
"transposed_report.tex\n",
"transposed_report.tsv\n",
"transposed_report.txt\n"
]
}
],
"source": [
"%%bash\n",
"ls /mnt/owl/Athaliana/quast_results/results_2018_01_16_09_35_09/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}