{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "### Running in Docker container on Swoose\n", "\n", "Started Docker container with the following command:\n", "\n", "```docker run -p 8888:8888 -v /home/sam/data/pacbio_oly/:/home/data -v /mnt/owl/:/home/owl -it bioinformatics/bioinformatics:v0 /bin/bash```\n", "\n", "The command allows ```/home/sam/data/``` and ```mnt/owl``` (our server Owl) to be accessible to the Docker container.\n", "\n", "Once access to Jupyter Notebook over port 8888 and makes my Jupyter Notebook GitHub repo and my data files the container was started, started Jupyter Notebook with the following command inside the Docker container:\n", "\n", "```jupyter notebook --allow-root```\n", "\n", "This is configured in the Docker container to launch a Jupyter Notebook without a browser on port 8888.\n", "The Docker container is running on an image created from this [Dockerfile (Git commit 63d36ab)](https://github.com/RobertsLab/code/commit/63d36ab221fc28124a70508aa1853f237928135c)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TODAY'S DATE\n", "Mon Oct 23 20:06:31 UTC 2017\n", "\n", "------------\n", "Computer Specs:\n", "\n", "Architecture: x86_64\n", "CPU op-mode(s): 32-bit, 64-bit\n", "Byte Order: Little Endian\n", "CPU(s): 24\n", "On-line CPU(s) list: 0-23\n", "Thread(s) per core: 2\n", "Core(s) per socket: 6\n", "Socket(s): 2\n", "NUMA node(s): 1\n", "Vendor ID: GenuineIntel\n", "CPU family: 6\n", "Model: 44\n", "Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz\n", "Stepping: 2\n", "CPU MHz: 2925.850\n", "BogoMIPS: 5851.96\n", "Virtualization: VT-x\n", "L1d cache: 32K\n", "L1i cache: 32K\n", "L2 cache: 256K\n", "L3 cache: 12288K\n", "NUMA node0 CPU(s): 0-23\n", "\n", "------------\n", "\n", "Memory Specs\n", "\n", " total used free shared buffers cached\n", "Mem: 70G 70G 588M 244M 387M 61G\n", "-/+ buffers/cache: 8.2G 62G\n", "Swap: 4.7G 1.1G 3.6G\n" ] } ], "source": [ "%%bash\n", "echo \"TODAY'S DATE\"\n", "date\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"Computer Specs:\"\n", "echo \"\"\n", "lscpu\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"\"\n", "echo \"Memory Specs\"\n", "echo \"\"\n", "free -mh" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1d27ba8eb17b\n" ] } ], "source": [ "%%bash\n", "hostname" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/data\n" ] } ], "source": [ "%%bash\n", "pwd" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Print list of Illumina files for easy copying/pasting into redundans command" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz\n" ] } ], "source": [ "%%bash\n", "for i in 1[56]*\n", "do\n", "echo \"$i\"\n", "done" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Print list of PacBio files for easy copying/pasting into redundans command" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "m*\n" ] } ], "source": [ "%%bash\n", "cd ..\n", "for i in m*\n", "do\n", "echo \"$i\"\n", "done" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/data\n" ] } ], "source": [ "cd /home/data" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz\n", "m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz\n", "m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz\n", "m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz\n", "m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz\n", "m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz\n", "m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz\n", "m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz\n", "m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz\n", "m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz\n", "m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz\n" ] } ], "source": [ "%%bash\n", "for i in m*\n", "do\n", "echo \"$i\"\n", "done" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "mkdir 20171023_oly_redundans_01" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/data\n" ] } ], "source": [ "cd /home/data/" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This notebook crashed while running yesterday. Updated the Dockerfile used to create this container and restarted today." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TODAY'S DATE\n", "Tue Oct 24 15:10:23 UTC 2017\n", "\n", "------------\n", "Computer Specs:\n", "\n", "Architecture: x86_64\n", "CPU op-mode(s): 32-bit, 64-bit\n", "Byte Order: Little Endian\n", "CPU(s): 24\n", "On-line CPU(s) list: 0-23\n", "Thread(s) per core: 2\n", "Core(s) per socket: 6\n", "Socket(s): 2\n", "NUMA node(s): 1\n", "Vendor ID: GenuineIntel\n", "CPU family: 6\n", "Model: 44\n", "Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz\n", "Stepping: 2\n", "CPU MHz: 2925.850\n", "BogoMIPS: 5851.96\n", "Virtualization: VT-x\n", "L1d cache: 32K\n", "L1i cache: 32K\n", "L2 cache: 256K\n", "L3 cache: 12288K\n", "NUMA node0 CPU(s): 0-23\n", "\n", "------------\n", "\n", "Memory Specs\n", "\n", " total used free shared buffers cached\n", "Mem: 70G 48G 22G 265M 1.0G 38G\n", "-/+ buffers/cache: 8.3G 62G\n", "Swap: 4.7G 1.0G 3.6G\n" ] } ], "source": [ "%%bash\n", "echo \"TODAY'S DATE\"\n", "date\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"Computer Specs:\"\n", "echo \"\"\n", "lscpu\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"\"\n", "echo \"Memory Specs\"\n", "echo \"\"\n", "free -mh" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5db143d2402e\n" ] } ], "source": [ "%%bash\n", "hostname" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "20171023_docker_oly_redundans.ipynb\r\n", "\u001b[0m\u001b[01;34m20171023_docker_oly_redundans_01\u001b[0m/\r\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz\r\n", "\u001b[01;32mm170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n" ] } ], "source": [ "ls" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "20171018_oly_pacbio.contigs.fasta\r\n", "20171023_docker_oly_redundans.ipynb\r\n", "\u001b[0m\u001b[01;34m20171023_docker_oly_redundans_01\u001b[0m/\r\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz\r\n", "\u001b[01;32mm170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n" ] } ], "source": [ "ls" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Traceback (most recent call last):\r\n", " File \"/usr/local/bioinformatics/redundans/redundans.py\", line 22, in \r\n", " import commands, os, resource, sys\r\n", "ModuleNotFoundError: No module named 'commands'\r\n" ] } ], "source": [ "!python /usr/local/bioinformatics/redundans/redundans.py \\\n", "-t 24 \\\n", "-l \\\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz \\\n", "m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz \\\n", "m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz \\\n", "-i \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "-f \\\n", "20171018_oly_pacbio.contigs.fasta \\\n", "-o \\\n", "/home/data/20171023_docker_oly_redundans_01/" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Traceback (most recent call last):\r\n", " File \"/usr/local/bioinformatics/redundans/redundans.py\", line 22, in \r\n", " import commands, os, resource, sys\r\n", "ModuleNotFoundError: No module named 'commands'\r\n" ] } ], "source": [ "!python /usr/local/bioinformatics/redundans/redundans.py \\\n", "-t 24 \\\n", "-l \\\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz \\\n", "m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz \\\n", "m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz \\\n", "-i \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "-f \\\n", "20171018_oly_pacbio.contigs.fasta \\\n", "-o \\\n", "/home/data/20171023_docker_oly_redundans_01/" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Some searching reveals that \"commands\" is no longer available in Python3; need to use Python2..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Running in Docker container on Swoose\n", "\n", "Started Docker container with the following command:\n", "\n", "```docker run -p 8888:8888 -v /home/sam/data/pacbio_oly/:/home/data -v /mnt/owl/:/home/owl -it bioinformatics/bioinformatics:v0 /bin/bash```\n", "\n", "The command allows ```/home/sam/data/``` to be accessible to the Docker container.\n", "\n", "Once access to Jupyter Notebook over port 8888 and makes my Jupyter Notebook GitHub repo and my data files the container was started, started Jupyter Notebook with the following command inside the Docker container:\n", "\n", "```jupyter notebook --allow-root```\n", "\n", "This is configured in the Docker container to launch a Jupyter Notebook without a browser on port 8888.\n", "The Docker container is running on an image created from this [Dockerfile (Git commit e38d24bd)](https://github.com/RobertsLab/code/commit/e38d24bd59d7c87958816a17a60975715952e26c)\n", "\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TODAY'S DATE\n", "Tue Oct 24 17:16:40 UTC 2017\n", "\n", "------------\n", "Computer Specs:\n", "\n", "Architecture: x86_64\n", "CPU op-mode(s): 32-bit, 64-bit\n", "Byte Order: Little Endian\n", "CPU(s): 24\n", "On-line CPU(s) list: 0-23\n", "Thread(s) per core: 2\n", "Core(s) per socket: 6\n", "Socket(s): 2\n", "NUMA node(s): 1\n", "Vendor ID: GenuineIntel\n", "CPU family: 6\n", "Model: 44\n", "Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz\n", "Stepping: 2\n", "CPU MHz: 2925.850\n", "BogoMIPS: 5851.96\n", "Virtualization: VT-x\n", "L1d cache: 32K\n", "L1i cache: 32K\n", "L2 cache: 256K\n", "L3 cache: 12288K\n", "NUMA node0 CPU(s): 0-23\n", "\n", "------------\n", "\n", "Memory Specs\n", "\n", " total used free shared buffers cached\n", "Mem: 70G 61G 9.5G 454M 1.2G 51G\n", "-/+ buffers/cache: 8.8G 61G\n", "Swap: 4.7G 1.0G 3.6G\n" ] } ], "source": [ "%%bash\n", "echo \"TODAY'S DATE\"\n", "date\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"Computer Specs:\"\n", "echo \"\"\n", "lscpu\n", "echo \"\"\n", "echo \"------------\"\n", "echo \"\"\n", "echo \"Memory Specs\"\n", "echo \"\"\n", "free -mh" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "8358b3f2ee6e\n" ] } ], "source": [ "%%bash\n", "hostname" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz\r\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz\r\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz\r\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz\r\n", "20171018_oly_pacbio.contigs.fasta\r\n", "20171023_docker_oly_redundans.ipynb\r\n", "\u001b[0m\u001b[01;34m20171023_docker_oly_redundans_01\u001b[0m/\r\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz\r\n", "\u001b[01;32mm170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n", "\u001b[01;32mm170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz\u001b[0m\u001b[K*\r\n" ] } ], "source": [ "ls" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Directory /home/data/20171023_docker_oly_redundans_01/ exists!\r\n" ] } ], "source": [ "!python /usr/local/bioinformatics/redundans/redundans.py \\\n", "-t 24 \\\n", "-l \\\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz \\\n", "m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz \\\n", "m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz \\\n", "-i \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "-f \\\n", "20171018_oly_pacbio.contigs.fasta \\\n", "-o \\\n", "/home/data/20171023_docker_oly_redundans_01/" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "rmdir 20171023_docker_oly_redundans_01/" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!python /usr/local/bioinformatics/redundans/redundans.py \\\n", "-t 24 \\\n", "-l \\\n", "m130619_081336_42134_c100525122550000001823081109281326_s1_p0.fastq.gz \\\n", "m170211_224036_42134_c101073082550000001823236402101737_s1_X0_filtered_subreads.fastq.gz \\\n", "m170301_100013_42134_c101174162550000001823269408211761_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_162825_42134_c101174162550000001823269408211762_s1_p0_filtered_subreads.fastq.gz \\\n", "m170301_225711_42134_c101174162550000001823269408211763_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_163922_42134_c101174252550000001823269408211742_s1_p0_filtered_subreads.fastq.gz \\\n", "m170308_230815_42134_c101174252550000001823269408211743_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_001112_42134_c101169372550000001823273008151717_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_063041_42134_c101169382550000001823273008151700_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_124938_42134_c101169382550000001823273008151701_s1_p0_filtered_subreads.fastq.gz \\\n", "m170315_190851_42134_c101169382550000001823273008151702_s1_p0_filtered_subreads.fastq.gz \\\n", "-i \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz \\\n", "151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \\\n", "151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz \\\n", "160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz \\\n", "-f \\\n", "20171018_oly_pacbio.contigs.fasta \\\n", "-o \\\n", "/home/data/20171023_docker_oly_redundans_01/" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.11" } }, "nbformat": 4, "nbformat_minor": 2 }