{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TODAY'S DATE:\n", "Mon Sep 10 10:03:21 PDT 2018\n", "------------\n", "\n", "Distributor ID:\tUbuntu\n", "Description:\tUbuntu 16.04.5 LTS\n", "Release:\t16.04\n", "Codename:\txenial\n", "\n", "------------\n", "HOSTNAME: \n", "roadrunner\n", "\n", "------------\n", "Computer Specs:\n", "\n", "Architecture: x86_64\n", "CPU op-mode(s): 32-bit, 64-bit\n", "Byte Order: Little Endian\n", "CPU(s): 16\n", "On-line CPU(s) list: 0-15\n", "Thread(s) per core: 2\n", "Core(s) per socket: 4\n", "Socket(s): 2\n", "NUMA node(s): 1\n", "Vendor ID: GenuineIntel\n", "CPU family: 6\n", "Model: 26\n", "Model name: Intel(R) Xeon(R) CPU E5520 @ 2.27GHz\n", "Stepping: 5\n", "CPU MHz: 2394.000\n", "CPU max MHz: 2394.0000\n", "CPU min MHz: 1596.0000\n", "BogoMIPS: 4521.80\n", "Virtualization: VT-x\n", "L1d cache: 32K\n", "L1i cache: 32K\n", "L2 cache: 256K\n", "L3 cache: 8192K\n", "NUMA node0 CPU(s): 0-15\n", "Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl xtopology nonstop_tsc aperfmperf eagerfpu pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm dca sse4_1 sse4_2 popcnt lahf_lm kaiser tpr_shadow vnmi flexpriority ept vpid dtherm ida\n", "\n", "------------\n", "\n", "Memory Specs\n", "\n", " total used free shared buff/cache available\n", "Mem: 47G 2.4G 36G 567M 8.7G 43G\n", "Swap: 47G 0B 47G\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "No LSB modules are available.\n" ] } ], "source": [ "%%bash\n", "echo \"TODAY'S DATE:\"\n", "date\n", "echo \"------------\"\n", "echo \"\"\n", "#Display operating system info\n", "lsb_release -a\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"HOSTNAME: \"; hostname \n", "echo \"\"\n", "echo \"------------\"\n", "echo \"Computer Specs:\"\n", "echo \"\"\n", "lscpu\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"\"\n", "echo \"Memory Specs\"\n", "echo \"\"\n", "free -mh" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Download necessary files" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "wget: missing URL\n", "Usage: wget [OPTION]... [URL]...\n", "\n", "Try `wget --help' for more options.\n", "\n", "real\t0m0.003s\n", "user\t0m0.000s\n", "sys\t0m0.000s\n", "bash: line 10: http://owl.fish.washington.edu/nightingales/C_virginica/: No such file or directory\n" ] } ], "source": [ "%%bash\n", "mkdir /home/sam/data/Cvirginica\n", "cd /home/sam/data/Cvirginica\n", "time \\\n", "wget \\\n", "--quiet \\\n", "--no-directories \\\n", "--recursive \\\n", "--accept gz \\\n", "--accept-regex \"2112_lane1_[ATCG]\"\n", "http://owl.fish.washington.edu/nightingales/C_virginica/\n", "sed '/^Subject:/ s/ / virginica download JOB COMPLETE/' ~/.default-subject.mail | msmtp \"$EMAIL\"\n", "\n", "ls -ltrh" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Fix typo - forgot line continuation slash after regex line." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 14G\n", "-rw-rw-r-- 1 sam sam 117M Apr 13 2015 2112_lane1_ACAGTG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 872M Apr 13 2015 2112_lane1_TTAGGC_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_TTAGGC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_GCCAAT_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_CAGATC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 133M Apr 13 2015 2112_lane1_ATCACG_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 690M Apr 13 2015 2112_lane1_CAGATC_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 704M Apr 13 2015 2112_lane1_TGACCA_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 789M Apr 13 2015 2112_lane1_GCCAAT_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_CAGATC_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_ACAGTG_L001_R1_001.fastq.gz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "real\t7m52.814s\n", "user\t0m10.672s\n", "sys\t1m4.880s\n" ] } ], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica\n", "time \\\n", "wget \\\n", "--quiet \\\n", "--no-directories \\\n", "--recursive \\\n", "--accept gz \\\n", "--accept-regex \"2112_lane1_[ATCG]\" \\\n", "http://owl.fish.washington.edu/nightingales/C_virginica/\n", "sed '/^Subject:/ s/ / virginica download JOB COMPLETE/' ~/.default-subject.mail | msmtp \"$EMAIL\"\n", "\n", "ls -ltrh" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 14G\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_ACAGTG_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 117M Apr 13 2015 2112_lane1_ACAGTG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 133M Apr 13 2015 2112_lane1_ATCACG_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_CAGATC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_CAGATC_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 690M Apr 13 2015 2112_lane1_CAGATC_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_GCCAAT_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 789M Apr 13 2015 2112_lane1_GCCAAT_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 704M Apr 13 2015 2112_lane1_TGACCA_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_TTAGGC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 872M Apr 13 2015 2112_lane1_TTAGGC_L001_R1_002.fastq.gz\n" ] } ], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica\n", "ls -lh" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Concatenate files\n", "\n", "Also renamed one file to maintain same naming structure as concatenated files." ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica\n", "cat \\\n", "2112_lane1_ACAGTG_L001_R1_001.fastq.gz \\\n", "2112_lane1_ACAGTG_L001_R1_002.fastq.gz \\\n", "> 2112_lane1_ACAGTG.fastq.gz\n", "\n", "cat \\\n", "2112_lane1_ATCACG_L001_R1_001.fastq.gz \\\n", "2112_lane1_ATCACG_L001_R1_002.fastq.gz \\\n", "2112_lane1_ATCACG_L001_R1_003.fastq.gz \\\n", "> 2112_lane1_ATCACG.fastq.gz\n", "\n", "cat \\\n", "2112_lane1_CAGATC_L001_R1_001.fastq.gz \\\n", "2112_lane1_CAGATC_L001_R1_002.fastq.gz \\\n", "2112_lane1_CAGATC_L001_R1_003.fastq.gz \\\n", "> 2112_lane1_CAGATC.fastq.gz\n", "\n", "cat \\\n", "2112_lane1_GCCAAT_L001_R1_001.fastq.gz \\\n", "2112_lane1_GCCAAT_L001_R1_002.fastq.gz \\\n", "> 2112_lane1_GCCAAT.fastq.gz\n", "\n", "mv \\\n", "2112_lane1_TGACCA_L001_R1_001.fastq.gz \\\n", "2112_lane1_TGACCA.fastq.gz\n", "\n", "cat \\\n", "2112_lane1_TTAGGC_L001_R1_001.fastq.gz \\\n", "2112_lane1_TTAGGC_L001_R1_002.fastq.gz \\\n", "> 2112_lane1_TTAGGC.fastq.gz\n", "\n", "sed '/^Subject:/ s/ / concatenation JOB COMPLETE/' ~/.default-subject.mail | msmtp \"$EMAIL\"" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 26G\n", "-rw-rw-r-- 1 sam sam 117M Apr 13 2015 2112_lane1_ACAGTG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 872M Apr 13 2015 2112_lane1_TTAGGC_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_TTAGGC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_GCCAAT_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_CAGATC_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Apr 13 2015 2112_lane1_ATCACG_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 133M Apr 13 2015 2112_lane1_ATCACG_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 690M Apr 13 2015 2112_lane1_CAGATC_L001_R1_003.fastq.gz\n", "-rw-rw-r-- 1 sam sam 704M Apr 13 2015 2112_lane1_TGACCA.fastq.gz\n", "-rw-rw-r-- 1 sam sam 789M Apr 13 2015 2112_lane1_GCCAAT_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_CAGATC_L001_R1_002.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.4G Apr 13 2015 2112_lane1_ACAGTG_L001_R1_001.fastq.gz\n", "-rw-rw-r-- 1 sam sam 1.5G Sep 10 10:59 2112_lane1_ACAGTG.fastq.gz\n", "-rw-rw-r-- 1 sam sam 3.1G Sep 10 10:59 2112_lane1_ATCACG.fastq.gz\n", "-rw-rw-r-- 1 sam sam 3.5G Sep 10 10:59 2112_lane1_CAGATC.fastq.gz\n", "-rw-rw-r-- 1 sam sam 2.3G Sep 10 10:59 2112_lane1_GCCAAT.fastq.gz\n", "-rw-rw-r-- 1 sam sam 2.3G Sep 10 10:59 2112_lane1_TTAGGC.fastq.gz\n" ] } ], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica\n", "ls -lhtr" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Move files around to improve organization" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total 14G\n", "-rw-rw-r-- 1 sam sam 1.5G Sep 10 10:59 2112_lane1_ACAGTG.fastq.gz\n", "-rw-rw-r-- 1 sam sam 3.1G Sep 10 10:59 2112_lane1_ATCACG.fastq.gz\n", "-rw-rw-r-- 1 sam sam 3.5G Sep 10 10:59 2112_lane1_CAGATC.fastq.gz\n", "-rw-rw-r-- 1 sam sam 2.3G Sep 10 10:59 2112_lane1_GCCAAT.fastq.gz\n", "-rw-rw-r-- 1 sam sam 704M Apr 13 2015 2112_lane1_TGACCA.fastq.gz\n", "-rw-rw-r-- 1 sam sam 2.3G Sep 10 10:59 2112_lane1_TTAGGC.fastq.gz\n" ] } ], "source": [ "%%bash\n", "data=/home/sam/data/Cvirginica/\n", "data_cat=/home/sam/data/Cvirginica/concatenated\n", "mkdir $data/concatenated\n", "mv $data/2112_lane1_TGACCA.fastq.gz $data_cat/\n", "mv $data/2112_lane1_ACAGTG.fastq.gz $data_cat/\n", "mv $data/2112_lane1_ATCACG.fastq.gz $data_cat/\n", "mv $data/2112_lane1_CAGATC.fastq.gz $data_cat/\n", "mv $data/2112_lane1_GCCAAT.fastq.gz $data_cat/\n", "mv $data/2112_lane1_TTAGGC.fastq.gz $data_cat/\n", "\n", "ls -lh $data_cat" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Run FastQC\n", "\n", "The code belows creates a space-delimited list of the FastQ files (FASTQ_LIST).\n", "\n", "This is then passed to FastQC." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\n", "real\t7m47.963s\n", "user\t28m26.772s\n", "sys\t0m42.752s\n" ] } ], "source": [ "%%bash\n", "data_cat=/home/sam/data/Cvirginica/concatenated\n", "mkdir $data_cat/20180910_Cvirginica_oil_fastqc\n", "cd $data_cat\n", "\n", "FASTQ_LIST=\"$(ls -1 *.gz| tr '\\n' ' ')\"\n", "time \\\n", "/home/shared/fastqc_v0.11.7/fastqc \\\n", "--extract \\\n", "--threads 16 \\\n", "--quiet \\\n", "--outdir $data_cat/20180910_Cvirginica_oil_fastqc \\\n", "$FASTQ_LIST\n", "\n", "sed '/^Subject:/ s/ / fastqc JOB COMPLETE/' ~/.default-subject.mail | msmtp \"$EMAIL\"" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2112_lane1_ACAGTG_fastqc\n", "2112_lane1_ACAGTG_fastqc.html\n", "2112_lane1_ACAGTG_fastqc.zip\n", "2112_lane1_ATCACG_fastqc\n", "2112_lane1_ATCACG_fastqc.html\n", "2112_lane1_ATCACG_fastqc.zip\n", "2112_lane1_CAGATC_fastqc\n", "2112_lane1_CAGATC_fastqc.html\n", "2112_lane1_CAGATC_fastqc.zip\n", "2112_lane1_GCCAAT_fastqc\n", "2112_lane1_GCCAAT_fastqc.html\n", "2112_lane1_GCCAAT_fastqc.zip\n", "2112_lane1_TGACCA_fastqc\n", "2112_lane1_TGACCA_fastqc.html\n", "2112_lane1_TGACCA_fastqc.zip\n", "2112_lane1_TTAGGC_fastqc\n", "2112_lane1_TTAGGC_fastqc.html\n", "2112_lane1_TTAGGC_fastqc.zip\n" ] } ], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica/concatenated/20180910_Cvirginica_oil_fastqc/\n", "ls" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Run MultiQC." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Searching 114 files..\n", "2112_lane1_ACAGTG_fastqc\n", "2112_lane1_ACAGTG_fastqc.html\n", "2112_lane1_ACAGTG_fastqc.zip\n", "2112_lane1_ATCACG_fastqc\n", "2112_lane1_ATCACG_fastqc.html\n", "2112_lane1_ATCACG_fastqc.zip\n", "2112_lane1_CAGATC_fastqc\n", "2112_lane1_CAGATC_fastqc.html\n", "2112_lane1_CAGATC_fastqc.zip\n", "2112_lane1_GCCAAT_fastqc\n", "2112_lane1_GCCAAT_fastqc.html\n", "2112_lane1_GCCAAT_fastqc.zip\n", "2112_lane1_TGACCA_fastqc\n", "2112_lane1_TGACCA_fastqc.html\n", "2112_lane1_TGACCA_fastqc.zip\n", "2112_lane1_TTAGGC_fastqc\n", "2112_lane1_TTAGGC_fastqc.html\n", "2112_lane1_TTAGGC_fastqc.zip\n", "multiqc_data\n", "multiqc_report.html\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[WARNING] multiqc : MultiQC Version v1.6 now available!\n", "[INFO ] multiqc : This is MultiQC v1.5.dev0\n", "[INFO ] multiqc : Template : default\n", "[INFO ] multiqc : Searching '.'\n", "[INFO ] fastqc : Found 6 reports\n", "[INFO ] multiqc : Compressing plot data\n", "[INFO ] multiqc : Report : multiqc_report.html\n", "[INFO ] multiqc : Data : multiqc_data\n", "[INFO ] multiqc : MultiQC complete\n", "\n", "real\t0m7.913s\n", "user\t0m3.312s\n", "sys\t0m0.220s\n" ] } ], "source": [ "%%bash\n", "cd /home/sam/data/Cvirginica/concatenated/20180910_Cvirginica_oil_fastqc/\n", "time \\\n", "multiqc .\n", "ls" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Files copied to Owl.\n", "\n", "Performed outside of notebook, due to ```sudo``` requirement." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2112_lane1_ACAGTG_fastqc\n", "2112_lane1_ACAGTG_fastqc.html\n", "2112_lane1_ACAGTG_fastqc.zip\n", "2112_lane1_ACAGTG.fastq.gz\n", "2112_lane1_ATCACG_fastqc\n", "2112_lane1_ATCACG_fastqc.html\n", "2112_lane1_ATCACG_fastqc.zip\n", "2112_lane1_ATCACG.fastq.gz\n", "2112_lane1_CAGATC_fastqc\n", "2112_lane1_CAGATC_fastqc.html\n", "2112_lane1_CAGATC_fastqc.zip\n", "2112_lane1_CAGATC.fastq.gz\n", "2112_lane1_GCCAAT_fastqc\n", "2112_lane1_GCCAAT_fastqc.html\n", "2112_lane1_GCCAAT_fastqc.zip\n", "2112_lane1_GCCAAT.fastq.gz\n", "2112_lane1_TGACCA_fastqc\n", "2112_lane1_TGACCA_fastqc.html\n", "2112_lane1_TGACCA_fastqc.zip\n", "2112_lane1_TGACCA.fastq.gz\n", "2112_lane1_TTAGGC_fastqc\n", "2112_lane1_TTAGGC_fastqc.html\n", "2112_lane1_TTAGGC_fastqc.zip\n", "2112_lane1_TTAGGC.fastq.gz\n", "multiqc_data\n", "multiqc_report.html\n" ] } ], "source": [ "%%bash\n", "ls /mnt/owl/Athaliana/20180910_Cvirginica_oil_fastqc/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.3" } }, "nbformat": 4, "nbformat_minor": 2 }